diff --git a/common/pom.xml b/common/pom.xml index aaeecc09556a76ced013a48ccfbdd7e782e6196b..67059462fa0ee12d6652a30f21846f4f70289dba 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -48,6 +48,11 @@ org.apache.hive hive-storage-api + + org.apache.hive + hive-serde-api + ${serde-api.version} + commons-cli diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java deleted file mode 100644 index 53684e7ab1b8d473ac7618ab2c830c226175a18c..0000000000000000000000000000000000000000 --- a/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -import org.apache.commons.lang.StringUtils; - -public abstract class HiveBaseChar { - protected String value; - - protected HiveBaseChar() { - } - - /** - * Sets the string value to a new value, obeying the max length defined for this object. - * @param val new value - */ - public void setValue(String val, int maxLength) { - value = HiveBaseChar.enforceMaxLength(val, maxLength); - } - - public void setValue(HiveBaseChar val, int maxLength) { - setValue(val.value, maxLength); - } - - public static String enforceMaxLength(String val, int maxLength) { - if (val == null) { - return null; - } - String value = val; - - if (maxLength > 0) { - int valLength = val.codePointCount(0, val.length()); - if (valLength > maxLength) { - // Truncate the excess chars to fit the character length. - // Also make sure we take supplementary chars into account. - value = val.substring(0, val.offsetByCodePoints(0, maxLength)); - } - } - return value; - } - - public static String getPaddedValue(String val, int maxLength) { - if (val == null) { - return null; - } - if (maxLength < 0) { - return val; - } - - int valLength = val.codePointCount(0, val.length()); - if (valLength > maxLength) { - return enforceMaxLength(val, maxLength); - } - - if (maxLength > valLength) { - // Make sure we pad the right amount of spaces; valLength is in terms of code points, - // while StringUtils.rpad() is based on the number of java chars. - int padLength = val.length() + (maxLength - valLength); - val = StringUtils.rightPad(val, padLength); - } - return val; - } - - public String getValue() { - return value; - } - - public int getCharacterLength() { - return value.codePointCount(0, value.length()); - } - - @Override - public int hashCode() { - return getValue().hashCode(); - } - - @Override - public String toString() { - return getValue(); - } -} diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java deleted file mode 100644 index 66aa524bb2c3b0835cdb8af3001ec6d96b5ae67c..0000000000000000000000000000000000000000 --- a/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -import org.apache.commons.lang.StringUtils; - -/** - * HiveChar. - * String values will be padded to full char length. - * Character legnth, comparison, hashCode should ignore trailing spaces. - */ -public class HiveChar extends HiveBaseChar - implements Comparable { - - public static final int MAX_CHAR_LENGTH = 255; - - public HiveChar() { - } - - public HiveChar(String val, int len) { - setValue(val, len); - } - - public HiveChar(HiveChar hc, int len) { - setValue(hc.value, len); - } - - /** - * Set char value, padding or truncating the value to the size of len parameter. - */ - public void setValue(String val, int len) { - super.setValue(HiveBaseChar.getPaddedValue(val, len), -1); - } - - public void setValue(String val) { - setValue(val, -1); - } - - public String getStrippedValue() { - return StringUtils.stripEnd(value, " "); - } - - public String getPaddedValue() { - return value; - } - - public int getCharacterLength() { - String strippedValue = getStrippedValue(); - return strippedValue.codePointCount(0, strippedValue.length()); - } - - public String toString() { - return getPaddedValue(); - } - - public int compareTo(HiveChar rhs) { - if (rhs == this) { - return 0; - } - return this.getStrippedValue().compareTo(rhs.getStrippedValue()); - } - - public boolean equals(Object rhs) { - if (rhs == this) { - return true; - } - if (rhs == null || rhs.getClass() != getClass()) { - return false; - } - return this.getStrippedValue().equals(((HiveChar) rhs).getStrippedValue()); - } - - public int hashCode() { - return getStrippedValue().hashCode(); - } -} diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java deleted file mode 100644 index 09009eb622ca938a718dcd8f7dc5343169c736c8..0000000000000000000000000000000000000000 --- a/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -/** - * - * HiveVarChar. - * String wrapper to support SQL VARCHAR features. - * Max string length is enforced. - * - */ -public class HiveVarchar extends HiveBaseChar - implements Comparable { - - public static final int MAX_VARCHAR_LENGTH = 65535; - - public HiveVarchar() { - } - - public HiveVarchar(String val, int len) { - setValue(val, len); - } - - public HiveVarchar(HiveVarchar hc, int len) { - setValue(hc, len); - } - - /** - * Set the new value - */ - public void setValue(String val) { - super.setValue(val, -1); - } - - public void setValue(HiveVarchar hc) { - super.setValue(hc.getValue(), -1); - } - - public int compareTo(HiveVarchar rhs) { - if (rhs == this) { - return 0; - } - return this.getValue().compareTo(rhs.getValue()); - } - - public boolean equals(Object rhs) { - if (rhs == this) { - return true; - } - return this.getValue().equals(((HiveVarchar)rhs).getValue()); - } -} diff --git a/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZUtil.java b/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZUtil.java index c49aefd5c201e37ee519be8dec1dd889d818d74f..90556aa88eee021a85d7e24ecaa4925b7f1af050 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZUtil.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZUtil.java @@ -133,18 +133,4 @@ public static TimestampTZ convert(Date date, ZoneId defaultTimeZone) { return parse(s, defaultTimeZone); } - public static ZoneId parseTimeZone(String timeZoneStr) { - if (timeZoneStr == null || timeZoneStr.trim().isEmpty() || - timeZoneStr.trim().toLowerCase().equals("local")) { - // default - return ZoneId.systemDefault(); - } - try { - return ZoneId.of(timeZoneStr); - } catch (DateTimeException e1) { - // default - throw new RuntimeException("Invalid time zone displacement value"); - } - } - } diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 711dfbdc1f05a25ffc64d297c6b1b25853d99a57..1fcb17ca48db10a2269bd4095f9942b86a658083 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate; -import org.apache.hadoop.hive.common.type.TimestampTZUtil; import org.apache.hadoop.hive.conf.Validator.PatternSet; import org.apache.hadoop.hive.conf.Validator.RangeValidator; import org.apache.hadoop.hive.conf.Validator.RatioValidator; @@ -34,6 +33,7 @@ import org.apache.hadoop.hive.conf.Validator.StringSet; import org.apache.hadoop.hive.conf.Validator.TimeValidator; import org.apache.hadoop.hive.conf.Validator.WritableDirectoryValidator; +import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat; @@ -4482,7 +4482,7 @@ private static String getSQLStdAuthDefaultWhiteListPattern() { */ public ZoneId getLocalTimeZone() { String timeZoneStr = getVar(ConfVars.HIVE_LOCAL_TIME_ZONE); - return TimestampTZUtil.parseTimeZone(timeZoneStr); + return TimestampLocalTZTypeInfo.parseTimeZone(timeZoneStr); } /** diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java index a66c13507abef42977dfdb315ff7d69404f67ac3..3df02f3ea7e298e185d57d8a19ee5bcf1065dcca 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java @@ -207,7 +207,7 @@ private static String determineFieldComment(String comment) { public static FieldSchema getFieldSchemaFromTypeInfo(String fieldName, TypeInfo typeInfo) { return new FieldSchema(fieldName, typeInfo.getTypeName(), - "generated by TypeInfoUtils.getFieldSchemaFromTypeInfo"); + "generated by StorageSchemaUtils.getFieldSchemaFromTypeInfo"); } } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/SerDeStorageSchemaReader.java b/metastore/src/java/org/apache/hadoop/hive/metastore/SerDeStorageSchemaReader.java deleted file mode 100644 index 59bcd5ca34d5083d357d7157abf3682399060a1a..0000000000000000000000000000000000000000 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/SerDeStorageSchemaReader.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.serde2.Deserializer; -import org.apache.hadoop.hive.metastore.api.EnvironmentContext; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.utils.StringUtils; - -import java.util.List; - -public class SerDeStorageSchemaReader implements StorageSchemaReader { - @Override - public List readSchema(Table tbl, EnvironmentContext envContext, Configuration conf) - throws MetaException { - ClassLoader orgHiveLoader = null; - try { - if (envContext != null) { - String addedJars = envContext.getProperties().get("hive.added.jars.path"); - if (org.apache.commons.lang.StringUtils.isNotBlank(addedJars)) { - //for thread safe - orgHiveLoader = conf.getClassLoader(); - ClassLoader loader = org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.addToClassPath( - orgHiveLoader, org.apache.commons.lang.StringUtils.split(addedJars, ",")); - conf.setClassLoader(loader); - } - } - - Deserializer s = HiveMetaStoreUtils.getDeserializer(conf, tbl, false); - return HiveMetaStoreUtils.getFieldsFromDeserializer(tbl.getTableName(), s); - } catch (Exception e) { - StringUtils.stringifyException(e); - throw new MetaException(e.getMessage()); - } finally { - if (orgHiveLoader != null) { - conf.setClassLoader(orgHiveLoader); - } - } - } -} diff --git a/pom.xml b/pom.xml index 6d1667585216ca2d329bc9d62901aac97caaba45..b9c265fa9bd113c7cd254fb94f156964261e217d 100644 --- a/pom.xml +++ b/pom.xml @@ -194,6 +194,7 @@ 1.7.10 4.0.4 3.0.0-SNAPSHOT + 3.0.0-SNAPSHOT 0.9.1-SNAPSHOT 0.92.0-incubating 2.2.0 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java index 1e044b4dcd5bc6dc03cb5757fc53d4b468771172..02bb82f2e74a1cba074c5a42f4be95cfc9d2e701 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveTypeEntry; /** * common class for reflective UDFs diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java index 05e21633ad34b744d2844b7a7fa4b308a2d3e55e..70243c09281888b9482ea9006d7b612f7c5cb7ba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java @@ -39,7 +39,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveTypeEntry; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; @@ -105,9 +105,9 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen return returnOI; } - private PrimitiveObjectInspectorUtils.PrimitiveTypeEntry getTypeFor(Class retType) + private PrimitiveTypeEntry getTypeFor(Class retType) throws UDFArgumentException { - PrimitiveObjectInspectorUtils.PrimitiveTypeEntry entry = + PrimitiveTypeEntry entry = PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaType(retType); if (entry == null) { entry = PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaClass(retType); diff --git a/serde/pom.xml b/serde/pom.xml index 0247c32452180aad73eb1932110096c1d044bd15..3ec09a2b028284fad9565e4f113487fef51eee5a 100644 --- a/serde/pom.xml +++ b/serde/pom.xml @@ -49,6 +49,11 @@ hive-shims ${project.version} + + org.apache.hive + hive-serde-api + ${storage-api.version} + com.google.code.findbugs diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java deleted file mode 100644 index a2a85b3dc44d7d7d676ad17eb476edbb5560f1cc..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2; - -import java.util.Map; -import java.util.Properties; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.io.Writable; - -import javax.annotation.Nullable; - -/** - * Abstract class for implementing SerDe. The abstract class has been created, so that - * new methods can be added in the underlying interface, SerDe, and only implementations - * that need those methods overwrite it. - */ -public abstract class AbstractSerDe implements Deserializer, Serializer { - - protected String configErrors; - - /** - * Initialize the SerDe. By default, this will use one set of properties, either the - * table properties or the partition properties. If a SerDe needs access to both sets, - * it should override this method. - * - * Eventually, once all SerDes have implemented this method, - * we should convert it to an abstract method. - * - * @param configuration Hadoop configuration - * @param tableProperties Table properties - * @param partitionProperties Partition properties - * @throws SerDeException - */ - public void initialize(Configuration configuration, Properties tableProperties, - Properties partitionProperties) throws SerDeException { - initialize(configuration, - SerDeUtils.createOverlayedProperties(tableProperties, partitionProperties)); - } - - /** - * Initialize the HiveSerializer. - * - * @param conf - * System properties. Can be null in compile time - * @param tbl - * table properties - * @throws SerDeException - */ - @Deprecated - public abstract void initialize(@Nullable Configuration conf, Properties tbl) - throws SerDeException; - - /** - * Returns the Writable class that would be returned by the serialize method. - * This is used to initialize SequenceFile header. - */ - public abstract Class getSerializedClass(); - - /** - * Serialize an object by navigating inside the Object with the - * ObjectInspector. In most cases, the return value of this function will be - * constant since the function will reuse the Writable object. If the client - * wants to keep a copy of the Writable, the client needs to clone the - * returned value. - */ - public abstract Writable serialize(Object obj, ObjectInspector objInspector) - throws SerDeException; - - /** - * Returns statistics collected when serializing - */ - public abstract SerDeStats getSerDeStats(); - - /** - * Deserialize an object out of a Writable blob. In most cases, the return - * value of this function will be constant since the function will reuse the - * returned object. If the client wants to keep a copy of the object, the - * client needs to clone the returned value by calling - * ObjectInspectorUtils.getStandardObject(). - * - * @param blob - * The Writable object containing a serialized object - * @return A Java object representing the contents in the blob. - */ - public abstract Object deserialize(Writable blob) throws SerDeException; - - /** - * Get the object inspector that can be used to navigate through the internal - * structure of the Object returned from deserialize(...). - */ - public abstract ObjectInspector getObjectInspector() throws SerDeException; - - /** - * Get the error messages during the Serde configuration - * - * @return The error messages in the configuration which are empty if no error occurred - */ - public String getConfigurationErrors() { - return configErrors == null ? "" : configErrors; - } - - /** - * @return Whether the SerDe that can store schema both inside and outside of metastore - * does, in fact, store it inside metastore, based on table parameters. - */ - public boolean shouldStoreFieldsInMetastore(Map tableParams) { - return false; // The default, unless SerDe overrides it. - } -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/Deserializer.java b/serde/src/java/org/apache/hadoop/hive/serde2/Deserializer.java deleted file mode 100644 index a1d3dd87665e2358209d92a4f9efcbe5c57418ea..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/Deserializer.java +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2; - -import java.util.Properties; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.io.Writable; - -/** - * HiveDeserializer is used to deserialize the data from hadoop Writable to a - * custom java object that can be of any type that the developer wants. - * - * HiveDeserializer also provides the ObjectInspector which can be used to - * inspect the internal structure of the object (that is returned by deserialize - * function). - * All deserializers should extend the abstract class AbstractDeserializer. - * The interface is necessary for SerDes to be able to implement both Serializer and Deserializer. - */ -public interface Deserializer { - - /** - * Initialize the HiveDeserializer. - * - * @param conf - * System properties - * @param tbl - * table properties - * @throws SerDeException - */ - void initialize(Configuration conf, Properties tbl) throws SerDeException; - - /** - * Deserialize an object out of a Writable blob. In most cases, the return - * value of this function will be constant since the function will reuse the - * returned object. If the client wants to keep a copy of the object, the - * client needs to clone the returned deserialized value by calling - * ObjectInspectorUtils.getStandardObject(). - * - * @param blob - * The Writable object containing a serialized object - * @return A Java object representing the contents in the blob. - */ - Object deserialize(Writable blob) throws SerDeException; - - /** - * Get the object inspector that can be used to navigate through the internal - * structure of the Object returned from deserialize(...). - */ - ObjectInspector getObjectInspector() throws SerDeException; - - /** - * Returns statistics collected when serializing - */ - SerDeStats getSerDeStats(); -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeException.java b/serde/src/java/org/apache/hadoop/hive/serde2/SerDeException.java deleted file mode 100644 index ea1ae9ca006754a19c0ab51d4d02be125476ee7e..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeException.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2; - -/** - * Generic exception class for SerDes. - * - */ - -public class SerDeException extends Exception { - private static final long serialVersionUID = 1L; - - public SerDeException() { - super(); - } - - public SerDeException(String message) { - super(message); - } - - public SerDeException(Throwable cause) { - super(cause); - } - - public SerDeException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeStats.java b/serde/src/java/org/apache/hadoop/hive/serde2/SerDeStats.java deleted file mode 100644 index 6cf2ccdcffff06bae21986c3df4c29fad8b560c5..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeStats.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2; - -public class SerDeStats { - - /** - * Class used to pass statistics information from serializer/deserializer to the tasks. - * A SerDeStats object is returned by calling SerDe.getStats(). - */ - - // currently we support only raw data size stat - private long rawDataSize; - private long rowCount; - - public SerDeStats() { - rawDataSize = 0; - rowCount = 0; - } - - /** - * Return the raw data size - * @return raw data size - */ - public long getRawDataSize() { - return rawDataSize; - } - - /** - * Set the raw data size - * @param uSize - size to be set - */ - public void setRawDataSize(long uSize) { - rawDataSize = uSize; - } - - /** - * Return the row count - * @return row count - */ - public long getRowCount() { - return rowCount; - } - - /** - * Set the row count - * @param rowCount - count of rows - */ - public void setRowCount(long rowCount) { - this.rowCount = rowCount; - } - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/Serializer.java b/serde/src/java/org/apache/hadoop/hive/serde2/Serializer.java deleted file mode 100644 index 3f07a86e31cdcb1ee082cda42163a192362d14ea..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/Serializer.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2; - -import java.util.Properties; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.io.Writable; - -/** - * HiveSerializer is used to serialize data to a Hadoop Writable object. The - * serialize In addition to the interface below, all implementations are assume - * to have a ctor that takes a single 'Table' object as argument. - * All serializers should extend the abstract class AbstractSerializer. - * The interface is necessary for SerDes to be able to implement both Serializer and Deserializer. - */ -public interface Serializer { - - /** - * Initialize the HiveSerializer. - * - * @param conf - * System properties - * @param tbl - * table properties - * @throws SerDeException - */ - void initialize(Configuration conf, Properties tbl) throws SerDeException; - - /** - * Returns the Writable class that would be returned by the serialize method. - * This is used to initialize SequenceFile header. - */ - Class getSerializedClass(); - - /** - * Serialize an object by navigating inside the Object with the - * ObjectInspector. In most cases, the return value of this function will be - * constant since the function will reuse the Writable object. If the client - * wants to keep a copy of the Writable, the client needs to clone the - * returned value. - */ - Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException; - - /** - * Returns statistics collected when serializing - */ - SerDeStats getSerDeStats(); -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java index 35d83bdb1af03df674842b72422e4b7d4d22b596..1998b94b26c3248d9806ff3646d4f50d38661c03 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java @@ -36,7 +36,7 @@ import java.util.Set; import org.apache.avro.Schema; -import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfoValidationUtils; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; @@ -143,7 +143,7 @@ public static TypeInfo generateTypeInfo(Schema schema, } try { - HiveDecimalUtils.validateParameter(precision, scale); + PrimitiveTypeInfoValidationUtils.validateParameter(precision, scale); } catch (Exception ex) { throw new AvroSerdeException("Invalid precision or scale for decimal type", ex); } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java index 0c8a4664e7e9454b5492ac9e3067c5211a49e64c..37675463054a371d613bd86de376859a0731ed93 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java @@ -36,7 +36,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveTypeEntry; import org.apache.hadoop.hive.serde2.thrift.ConfigurableTProtocol; import org.apache.hadoop.hive.serde2.thrift.TReflectionUtils; import org.apache.hadoop.io.BytesWritable; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveBaseCharWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveBaseCharWritable.java deleted file mode 100644 index 8c37a9bb75f2e425099461fd28397f1dceed97b4..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveBaseCharWritable.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.io; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.io.Text; -import org.apache.hive.common.util.HiveStringUtils; - -public abstract class HiveBaseCharWritable { - protected Text value = new Text(); - - public HiveBaseCharWritable() { - } - - public int getCharacterLength() { - return HiveStringUtils.getTextUtfLength(value); - } - - /** - * Access to the internal Text member. Use with care. - * @return - */ - public Text getTextValue() { - return value; - } - - public void readFields(DataInput in) throws IOException { - value.readFields(in); - } - - public void write(DataOutput out) throws IOException { - value.write(out); - } - - public boolean equals(Object obj) { - if (obj == null || (obj.getClass() != this.getClass())) { - return false; - } - return value.equals(((HiveBaseCharWritable)obj).value); - } - - public int hashCode() { - return value.hashCode(); - } -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveCharWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveCharWritable.java deleted file mode 100644 index 2236a8799980fa1ba925b7b2afb5e4b0dbaf073e..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveCharWritable.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.io; - -import org.apache.hadoop.hive.common.type.HiveBaseChar; -import org.apache.hadoop.hive.common.type.HiveChar; -import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hive.common.util.HiveStringUtils; - -/** - * HiveCharWritable. - * String values will be padded to full char length. - * Character legnth, comparison, hashCode should ignore trailing spaces. - */ -public class HiveCharWritable extends HiveBaseCharWritable - implements WritableComparable { - - public HiveCharWritable() { - } - - public HiveCharWritable(HiveChar hc) { - set(hc); - } - - public HiveCharWritable(HiveCharWritable hcw) { - set(hcw); - } - - public void set(HiveChar val) { - set(val.getValue(), -1); - } - - public void set(String val) { - set(val, -1); - } - - public void set(HiveCharWritable val) { - value.set(val.value); - } - - public void set(HiveCharWritable val, int maxLength) { - set(val.getHiveChar(), maxLength); - } - - public void set(HiveChar val, int len) { - set(val.getValue(), len); - } - - public void set(String val, int maxLength) { - value.set(HiveBaseChar.getPaddedValue(val, maxLength)); - } - - public HiveChar getHiveChar() { - // Copy string value as-is - return new HiveChar(value.toString(), -1); - } - - public void enforceMaxLength(int maxLength) { - if (getCharacterLength()!=maxLength) - set(getHiveChar(), maxLength); - } - - public Text getStrippedValue() { - // A lot of these methods could be done more efficiently by operating on the Text value - // directly, rather than converting to HiveChar. - return new Text(getHiveChar().getStrippedValue()); - } - - public Text getPaddedValue() { - return getTextValue(); - } - - public int getCharacterLength() { - return HiveStringUtils.getTextUtfLength(getStrippedValue()); - } - - public int compareTo(HiveCharWritable rhs) { - return getStrippedValue().compareTo(rhs.getStrippedValue()); - } - - public boolean equals(Object rhs) { - if (rhs == this) { - return true; - } - if (rhs == null || rhs.getClass() != getClass()) { - return false; - } - return this.getStrippedValue().equals(((HiveCharWritable) rhs).getStrippedValue()); - } - - public int hashCode() { - return getStrippedValue().hashCode(); - } - - @Override - public String toString() { - return getPaddedValue().toString(); - } -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java deleted file mode 100644 index 270d97fa6a856916f95006f0a29506dc9391d9aa..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.io; - -import org.apache.hadoop.hive.common.type.HiveBaseChar; -import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.io.WritableComparable; - -public class HiveVarcharWritable extends HiveBaseCharWritable - implements WritableComparable{ - - public HiveVarcharWritable() { - } - - public HiveVarcharWritable(HiveVarchar hc) { - set(hc); - } - - public HiveVarcharWritable(HiveVarcharWritable hcw) { - set(hcw); - } - - public void set(HiveVarchar val) { - set(val.getValue()); - } - - public void set(String val) { - set(val, -1); // copy entire string value - } - - public void set(HiveVarcharWritable val) { - value.set(val.value); - } - - public void set(HiveVarcharWritable val, int maxLength) { - set(val.getHiveVarchar(), maxLength); - } - - public void set(HiveVarchar val, int len) { - set(val.getValue(), len); - } - - public void set(String val, int maxLength) { - value.set(HiveBaseChar.enforceMaxLength(val, maxLength)); - } - - public HiveVarchar getHiveVarchar() { - return new HiveVarchar(value.toString(), -1); - } - - public void enforceMaxLength(int maxLength) { - if (getCharacterLength() > maxLength) { - set(value.toString(), maxLength); - } - } - - @Override - public int compareTo(HiveVarcharWritable rhs) { - return value.compareTo(rhs.value); - } - - @Override - public String toString() { - return value.toString(); - } -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ConstantObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ConstantObjectInspector.java deleted file mode 100644 index 1adc72b0897fa39f8144bb002ce17d5e47f0cce4..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ConstantObjectInspector.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.objectinspector; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; - -/** - * ConstantObjectInspector. This interface should be implemented by - * ObjectInspectors which represent constant values and can return them without - * an evaluation. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public interface ConstantObjectInspector extends ObjectInspector { - - Object getWritableConstantValue(); - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectInspector.java deleted file mode 100644 index 56d59e7089b15543f114c1a041b36644386fa776..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectInspector.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.objectinspector; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; - -import java.util.List; - -/** - * ListObjectInspector. - * - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public interface ListObjectInspector extends ObjectInspector { - - // ** Methods that does not need a data object ** - ObjectInspector getListElementObjectInspector(); - - // ** Methods that need a data object ** - /** - * returns null for null list, out-of-the-range index. - */ - Object getListElement(Object data, int index); - - /** - * returns -1 for data = null. - */ - int getListLength(Object data); - - /** - * returns null for data = null. - * - * Note: This method should not return a List object that is reused by the - * same ListObjectInspector, because it's possible that the same - * ListObjectInspector will be used in multiple places in the code. - * - * However it's OK if the List object is part of the Object data. - */ - List getList(Object data); - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapObjectInspector.java deleted file mode 100644 index 40b11f05f76db7dfbe2a6e4a17320aefbd7c374f..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapObjectInspector.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.objectinspector; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; - -import java.util.Map; - -/** - * MapObjectInspector. - * - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public interface MapObjectInspector extends ObjectInspector { - - // ** Methods that does not need a data object ** - // Map Type - ObjectInspector getMapKeyObjectInspector(); - - ObjectInspector getMapValueObjectInspector(); - - // ** Methods that need a data object ** - // In this function, key has to be of the same structure as the Map expects. - // Most cases key will be primitive type, so it's OK. - // In rare cases that key is not primitive, the user is responsible for - // defining - // the hashCode() and equals() methods of the key class. - Object getMapValueElement(Object data, Object key); - - /** - * returns null for data = null. - * - * Note: This method should not return a Map object that is reused by the same - * MapObjectInspector, because it's possible that the same MapObjectInspector - * will be used in multiple places in the code. - * - * However it's OK if the Map object is part of the Object data. - */ - Map getMap(Object data); - - /** - * returns -1 for NULL map. - */ - int getMapSize(Object data); -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java deleted file mode 100644 index 5ed429de737e33154f66b106a7a53dafbd179071..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.objectinspector; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; - -/** - * ObjectInspector helps us to look into the internal structure of a complex - * object. - * - * A (probably configured) ObjectInspector instance stands for a specific type - * and a specific way to store the data of that type in the memory. - * - * For native java Object, we can directly access the internal structure through - * member fields and methods. ObjectInspector is a way to delegate that - * functionality away from the Object, so that we have more control on the - * behavior of those actions. - * - * An efficient implementation of ObjectInspector should rely on factory, so - * that we can make sure the same ObjectInspector only has one instance. That - * also makes sure hashCode() and equals() methods of java.lang.Object directly - * works for ObjectInspector as well. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public interface ObjectInspector extends Cloneable { - - /** - * Category. - * - */ - public static enum Category { - PRIMITIVE, LIST, MAP, STRUCT, UNION - }; - - /** - * Returns the name of the data type that is inspected by this - * ObjectInspector. This is used to display the type information to the user. - * - * For primitive types, the type name is standardized. For other types, the - * type name can be something like "list<int>", "map<int,string>", java class - * names, or user-defined type names similar to typedef. - */ - String getTypeName(); - - /** - * An ObjectInspector must inherit from one of the following interfaces if - * getCategory() returns: PRIMITIVE: PrimitiveObjectInspector LIST: - * ListObjectInspector MAP: MapObjectInspector STRUCT: StructObjectInspector. - */ - Category getCategory(); -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java deleted file mode 100644 index 3c58f066f2c5c9748afe220b71867a0d4fc68470..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.objectinspector; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; - - -/** - * PrimitiveObjectInspector. - * - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public interface PrimitiveObjectInspector extends ObjectInspector { - - /** - * The primitive types supported by Hive. - */ - public static enum PrimitiveCategory { - VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, - DATE, TIMESTAMP, TIMESTAMPLOCALTZ, BINARY, DECIMAL, VARCHAR, CHAR, - INTERVAL_YEAR_MONTH, INTERVAL_DAY_TIME, UNKNOWN - }; - - public PrimitiveTypeInfo getTypeInfo(); - - /** - * Get the primitive category of the PrimitiveObjectInspector. - */ - PrimitiveCategory getPrimitiveCategory(); - - /** - * Get the Primitive Writable class which is the return type of - * getPrimitiveWritableObject() and copyToPrimitiveWritableObject(). - */ - Class getPrimitiveWritableClass(); - - /** - * Return the data in an instance of primitive writable Object. If the Object - * is already a primitive writable Object, just return o. - */ - Object getPrimitiveWritableObject(Object o); - - /** - * Get the Java Primitive class which is the return type of - * getJavaPrimitiveObject(). - */ - Class getJavaPrimitiveClass(); - - /** - * Get the Java Primitive object. - */ - Object getPrimitiveJavaObject(Object o); - - /** - * Get a copy of the Object in the same class, so the return value can be - * stored independently of the parameter. - * - * If the Object is a Primitive Java Object, we just return the parameter - * since Primitive Java Object is immutable. - */ - Object copyObject(Object o); - - /** - * Whether the ObjectInspector prefers to return a Primitive Writable Object - * instead of a Primitive Java Object. This can be useful for determining the - * most efficient way to getting data out of the Object. - */ - boolean preferWritable(); - - /** - * The precision of the underlying data. - */ - int precision(); - - /** - * The scale of the underlying data. - */ - int scale(); - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java deleted file mode 100644 index dc147d6565725817cd7411f224ca93cd648a6f9c..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.objectinspector; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; - -/** - * Classes implementing this interface are considered to represent a field of a - * struct for this serde package. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public interface StructField { - - /** - * Get the name of the field. The name should be always in lower-case. - */ - String getFieldName(); - - /** - * Get the ObjectInspector for the field. - */ - ObjectInspector getFieldObjectInspector(); - - /** - * Get the fieldID for the field. - */ - int getFieldID(); - - /** - * Get the comment for the field. May be null if no comment provided. - */ - String getFieldComment(); -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructObjectInspector.java deleted file mode 100644 index f7463543f35d876023a5426a95c9d256eff5660a..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructObjectInspector.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.objectinspector; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; - -import java.util.List; - -/** - * StructObjectInspector. - * - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public abstract class StructObjectInspector implements ObjectInspector { - - // ** Methods that does not need a data object ** - /** - * Returns all the fields. - */ - public abstract List getAllStructFieldRefs(); - - /** - * Look up a field. - */ - public abstract StructField getStructFieldRef(String fieldName); - - // ** Methods that need a data object ** - /** - * returns null for data = null. - */ - public abstract Object getStructFieldData(Object data, StructField fieldRef); - - /** - * returns null for data = null. - */ - public abstract List getStructFieldsDataAsList(Object data); - - public boolean isSettable() { - return false; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - List fields = getAllStructFieldRefs(); - sb.append(getClass().getName()); - sb.append("<"); - for (int i = 0; i < fields.size(); i++) { - if (i > 0) { - sb.append(","); - } - sb.append(fields.get(i).getFieldObjectInspector().toString()); - } - sb.append(">"); - return sb.toString(); - } -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObjectInspector.java deleted file mode 100644 index f05ea38cdc48c210b9564cb789e5b64dae264641..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObjectInspector.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.objectinspector; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; - -import java.util.List; - -/** - * UnionObjectInspector works on union data that is stored as UnionObject. - * - * It holds the list of the object inspectors corresponding to each type of the - * object the Union can hold. - * - * UnionObjectInspector. - * - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public interface UnionObjectInspector extends ObjectInspector { - - /** - * Returns the array of ObjectInspectors that are for each of the tags. - */ - List getObjectInspectors(); - - /** - * Return the tag of the object. - */ - byte getTag(Object o); - - /** - * Return the field based on the tag associated with the Object. - */ - Object getField(Object o); - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java index 2445e08af69366af6accca5797a80b112cde3c24..eab43f6558ffdc98dfb7b2180728dba3203186be 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java index 88e7fb00f4bae6bdd98abf2b5a6fba706609c892..e0bca2092155967bc4fe5bc74e2e939f8866d6ee 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java @@ -18,20 +18,13 @@ package org.apache.hadoop.hive.serde2.objectinspector.primitive; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; import java.nio.charset.CharacterCodingException; import java.nio.charset.StandardCharsets; import java.sql.Date; import java.sql.Timestamp; import java.time.DateTimeException; import java.time.ZoneId; -import java.util.HashMap; -import java.util.Map; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.common.type.TimestampTZ; import org.apache.hadoop.hive.common.type.TimestampTZUtil; import org.apache.hadoop.hive.ql.util.TimestampUtils; @@ -66,7 +59,6 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableUtils; /** * ObjectInspectorFactory is the primary way to create new ObjectInspector @@ -77,124 +69,6 @@ */ public final class PrimitiveObjectInspectorUtils { - /** - * TypeEntry stores information about a Hive Primitive TypeInfo. - */ - @InterfaceAudience.Public - @InterfaceStability.Stable - public static class PrimitiveTypeEntry implements Writable, Cloneable { - - /** - * The category of the PrimitiveType. - */ - public PrimitiveObjectInspector.PrimitiveCategory primitiveCategory; - - /** - * primitiveJavaType refers to java types like int, double, etc. - */ - public Class primitiveJavaType; - /** - * primitiveJavaClass refers to java classes like Integer, Double, String - * etc. - */ - public Class primitiveJavaClass; - /** - * writableClass refers to hadoop Writable classes like IntWritable, - * DoubleWritable, Text etc. - */ - public Class primitiveWritableClass; - /** - * typeName is the name of the type as in DDL. - */ - public String typeName; - - protected PrimitiveTypeEntry() { - super(); - } - - PrimitiveTypeEntry( - PrimitiveObjectInspector.PrimitiveCategory primitiveCategory, - String typeName, Class primitiveType, Class javaClass, - Class hiveClass) { - this.primitiveCategory = primitiveCategory; - primitiveJavaType = primitiveType; - primitiveJavaClass = javaClass; - primitiveWritableClass = hiveClass; - this.typeName = typeName; - } - - @Override - public void readFields(DataInput in) throws IOException { - primitiveCategory = WritableUtils.readEnum(in, - PrimitiveObjectInspector.PrimitiveCategory.class); - typeName = WritableUtils.readString(in); - try { - primitiveJavaType = Class.forName(WritableUtils.readString(in)); - primitiveJavaClass = Class.forName(WritableUtils.readString(in)); - primitiveWritableClass = Class.forName(WritableUtils.readString(in)); - } catch (ClassNotFoundException e) { - throw new IOException(e); - } - } - - @Override - public void write(DataOutput out) throws IOException { - - WritableUtils.writeEnum(out, primitiveCategory); - WritableUtils.writeString(out, typeName); - WritableUtils.writeString(out, primitiveJavaType.getName()); - WritableUtils.writeString(out, primitiveJavaClass.getName()); - WritableUtils.writeString(out, primitiveWritableClass.getName()); - } - - @Override - public Object clone() { - PrimitiveTypeEntry result = new PrimitiveTypeEntry( - this.primitiveCategory, - this.typeName, - this.primitiveJavaType, - this.primitiveJavaClass, - this.primitiveWritableClass); - return result; - } - - @Override - public String toString() { - return typeName; - } - - } - - static final Map primitiveCategoryToTypeEntry = new HashMap(); - static final Map, PrimitiveTypeEntry> primitiveJavaTypeToTypeEntry = new HashMap, PrimitiveTypeEntry>(); - static final Map, PrimitiveTypeEntry> primitiveJavaClassToTypeEntry = new HashMap, PrimitiveTypeEntry>(); - static final Map, PrimitiveTypeEntry> primitiveWritableClassToTypeEntry = new HashMap, PrimitiveTypeEntry>(); - - // Base type name to PrimitiveTypeEntry map. - private static final Map typeNameToTypeEntry = new HashMap(); - - static void addParameterizedType(PrimitiveTypeEntry t) { - typeNameToTypeEntry.put(t.toString(), t); - } - - static void registerType(PrimitiveTypeEntry t) { - if (t.primitiveCategory != PrimitiveCategory.UNKNOWN) { - primitiveCategoryToTypeEntry.put(t.primitiveCategory, t); - } - if (t.primitiveJavaType != null) { - primitiveJavaTypeToTypeEntry.put(t.primitiveJavaType, t); - } - if (t.primitiveJavaClass != null) { - primitiveJavaClassToTypeEntry.put(t.primitiveJavaClass, t); - } - if (t.primitiveWritableClass != null) { - primitiveWritableClassToTypeEntry.put(t.primitiveWritableClass, t); - } - if (t.typeName != null) { - typeNameToTypeEntry.put(t.typeName, t); - } - } - public static final PrimitiveTypeEntry binaryTypeEntry = new PrimitiveTypeEntry( PrimitiveCategory.BINARY, serdeConstants.BINARY_TYPE_NAME, byte[].class, byte[].class, BytesWritable.class); @@ -257,25 +131,25 @@ static void registerType(PrimitiveTypeEntry t) { PrimitiveCategory.UNKNOWN, "unknown", null, Object.class, null); static { - registerType(binaryTypeEntry); - registerType(stringTypeEntry); - registerType(charTypeEntry); - registerType(varcharTypeEntry); - registerType(booleanTypeEntry); - registerType(intTypeEntry); - registerType(longTypeEntry); - registerType(floatTypeEntry); - registerType(voidTypeEntry); - registerType(doubleTypeEntry); - registerType(byteTypeEntry); - registerType(shortTypeEntry); - registerType(dateTypeEntry); - registerType(timestampTypeEntry); - registerType(timestampTZTypeEntry); - registerType(intervalYearMonthTypeEntry); - registerType(intervalDayTimeTypeEntry); - registerType(decimalTypeEntry); - registerType(unknownTypeEntry); + PrimitiveTypeEntry.registerType(binaryTypeEntry); + PrimitiveTypeEntry.registerType(stringTypeEntry); + PrimitiveTypeEntry.registerType(charTypeEntry); + PrimitiveTypeEntry.registerType(varcharTypeEntry); + PrimitiveTypeEntry.registerType(booleanTypeEntry); + PrimitiveTypeEntry.registerType(intTypeEntry); + PrimitiveTypeEntry.registerType(longTypeEntry); + PrimitiveTypeEntry.registerType(floatTypeEntry); + PrimitiveTypeEntry.registerType(voidTypeEntry); + PrimitiveTypeEntry.registerType(doubleTypeEntry); + PrimitiveTypeEntry.registerType(byteTypeEntry); + PrimitiveTypeEntry.registerType(shortTypeEntry); + PrimitiveTypeEntry.registerType(dateTypeEntry); + PrimitiveTypeEntry.registerType(timestampTypeEntry); + PrimitiveTypeEntry.registerType(timestampTZTypeEntry); + PrimitiveTypeEntry.registerType(intervalYearMonthTypeEntry); + PrimitiveTypeEntry.registerType(intervalDayTimeTypeEntry); + PrimitiveTypeEntry.registerType(decimalTypeEntry); + PrimitiveTypeEntry.registerType(unknownTypeEntry); } /** @@ -283,7 +157,7 @@ static void registerType(PrimitiveTypeEntry t) { * class. */ public static Class primitiveJavaTypeToClass(Class clazz) { - PrimitiveTypeEntry t = primitiveJavaTypeToTypeEntry.get(clazz); + PrimitiveTypeEntry t = PrimitiveTypeEntry.fromJavaType(clazz); return t == null ? clazz : t.primitiveJavaClass; } @@ -291,38 +165,38 @@ static void registerType(PrimitiveTypeEntry t) { * Whether the class is a Java Primitive type or a Java Primitive class. */ public static boolean isPrimitiveJava(Class clazz) { - return primitiveJavaTypeToTypeEntry.get(clazz) != null - || primitiveJavaClassToTypeEntry.get(clazz) != null; + return PrimitiveTypeEntry.fromJavaType(clazz) != null + || PrimitiveTypeEntry.fromJavaClass(clazz) != null; } /** * Whether the class is a Java Primitive type. */ public static boolean isPrimitiveJavaType(Class clazz) { - return primitiveJavaTypeToTypeEntry.get(clazz) != null; + return PrimitiveTypeEntry.fromJavaType(clazz) != null; } /** * Whether the class is a Java Primitive class. */ public static boolean isPrimitiveJavaClass(Class clazz) { - return primitiveJavaClassToTypeEntry.get(clazz) != null; + return PrimitiveTypeEntry.fromJavaClass(clazz) != null; } /** * Whether the class is a Hive Primitive Writable class. */ public static boolean isPrimitiveWritableClass(Class clazz) { - return primitiveWritableClassToTypeEntry.get(clazz) != null; + return PrimitiveTypeEntry.fromWritableClass(clazz) != null; } /** * Get the typeName from a Java Primitive Type or Java PrimitiveClass. */ public static String getTypeNameFromPrimitiveJava(Class clazz) { - PrimitiveTypeEntry t = primitiveJavaTypeToTypeEntry.get(clazz); + PrimitiveTypeEntry t = PrimitiveTypeEntry.fromJavaType(clazz); if (t == null) { - t = primitiveJavaClassToTypeEntry.get(clazz); + t = PrimitiveTypeEntry.fromJavaClass(clazz); } return t == null ? null : t.typeName; } @@ -331,7 +205,7 @@ public static String getTypeNameFromPrimitiveJava(Class clazz) { * Get the typeName from a Primitive Writable Class. */ public static String getTypeNameFromPrimitiveWritable(Class clazz) { - PrimitiveTypeEntry t = primitiveWritableClassToTypeEntry.get(clazz); + PrimitiveTypeEntry t = PrimitiveTypeEntry.fromWritableClass(clazz); return t == null ? null : t.typeName; } @@ -340,16 +214,16 @@ public static String getTypeNameFromPrimitiveWritable(Class clazz) { */ public static PrimitiveTypeEntry getTypeEntryFromPrimitiveCategory( PrimitiveCategory category) { - return primitiveCategoryToTypeEntry.get(category); + return PrimitiveTypeEntry.fromPrimitiveCategory(category); } /** * Get the TypeEntry for a Java Primitive Type or Java PrimitiveClass. */ public static PrimitiveTypeEntry getTypeEntryFromPrimitiveJava(Class clazz) { - PrimitiveTypeEntry t = primitiveJavaTypeToTypeEntry.get(clazz); + PrimitiveTypeEntry t = PrimitiveTypeEntry.fromJavaType(clazz); if (t == null) { - t = primitiveJavaClassToTypeEntry.get(clazz); + t = PrimitiveTypeEntry.fromJavaClass(clazz); } return t; } @@ -359,7 +233,7 @@ public static PrimitiveTypeEntry getTypeEntryFromPrimitiveJava(Class clazz) { */ public static PrimitiveTypeEntry getTypeEntryFromPrimitiveJavaType( Class clazz) { - return primitiveJavaTypeToTypeEntry.get(clazz); + return PrimitiveTypeEntry.fromJavaType(clazz); } /** @@ -367,7 +241,7 @@ public static PrimitiveTypeEntry getTypeEntryFromPrimitiveJavaType( */ public static PrimitiveTypeEntry getTypeEntryFromPrimitiveJavaClass( Class clazz) { - return primitiveJavaClassToTypeEntry.get(clazz); + return PrimitiveTypeEntry.fromJavaClass(clazz); } /** @@ -375,14 +249,14 @@ public static PrimitiveTypeEntry getTypeEntryFromPrimitiveJavaClass( */ public static PrimitiveTypeEntry getTypeEntryFromPrimitiveWritableClass( Class clazz) { - return primitiveWritableClassToTypeEntry.get(clazz); + return PrimitiveTypeEntry.fromWritableClass(clazz); } /** * Get the TypeEntry for the given base type name (int, varchar, etc). */ public static PrimitiveTypeEntry getTypeEntryFromTypeName(String typeName) { - return typeNameToTypeEntry.get(typeName); + return PrimitiveTypeEntry.fromTypeName(typeName); } /** diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java deleted file mode 100644 index fc54a8f28cf4caac5f853e3568e6ac10cb9e564d..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; - -@InterfaceAudience.Public -@InterfaceStability.Stable -public abstract class BaseCharTypeInfo extends PrimitiveTypeInfo { - private static final long serialVersionUID = 1L; - - private int length; - - // no-arg constructor to make kyro happy. - public BaseCharTypeInfo() { - } - - public BaseCharTypeInfo(String typeName) { - super(typeName); - } - - public BaseCharTypeInfo(String typeName, int length) { - super(typeName); - this.length = length; - } - - public int getLength() { - return length; - } - - public void setLength(int length) { - this.length = length; - } - - @Override - public String getQualifiedName() { - return getQualifiedName(typeName, length); - } - - public static String getQualifiedName(String typeName, int length) { - StringBuilder sb = new StringBuilder(typeName); - sb.append("("); - sb.append(length); - sb.append(")"); - return sb.toString(); - } - - @Override - public void setTypeName(String typeName) { - // type name should already be set by subclass - return; - } -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharUtils.java index 09e07c749583eb32d7cd12bf0d5ebd7ab02bf99f..b52f6db3c41b2541b1b7aeec78e55b7bcd86508d 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharUtils.java @@ -27,20 +27,6 @@ public class BaseCharUtils { - public static void validateVarcharParameter(int length) { - if (length > HiveVarchar.MAX_VARCHAR_LENGTH || length < 1) { - throw new RuntimeException("Varchar length " + length + " out of allowed range [1, " + - HiveVarchar.MAX_VARCHAR_LENGTH + "]"); - } - } - - public static void validateCharParameter(int length) { - if (length > HiveChar.MAX_CHAR_LENGTH || length < 1) { - throw new RuntimeException("Char length " + length + " out of allowed range [1, " + - HiveChar.MAX_CHAR_LENGTH + "]"); - } - } - public static boolean doesWritableMatchTypeParams(HiveBaseCharWritable writable, BaseCharTypeInfo typeInfo) { return typeInfo.getLength() >= writable.getCharacterLength(); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java deleted file mode 100644 index 871c0f0f564ce3e0b898966983e47dae1ec886ca..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import org.apache.hadoop.hive.serde.serdeConstants; - -public class CharTypeInfo extends BaseCharTypeInfo { - private static final long serialVersionUID = 1L; - - // no-arg constructor to make kyro happy. - public CharTypeInfo() { - super(serdeConstants.CHAR_TYPE_NAME); - } - - public CharTypeInfo(int length) { - super(serdeConstants.CHAR_TYPE_NAME, length); - BaseCharUtils.validateCharParameter(length); - } - - @Override - public String getTypeName() { - return getQualifiedName(); - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (other == null || getClass() != other.getClass()) { - return false; - } - - CharTypeInfo pti = (CharTypeInfo) other; - - return this.typeName.equals(pti.typeName) && this.getLength() == pti.getLength(); - } - - /** - * Generate the hashCode for this TypeInfo. - */ - @Override - public int hashCode() { - return getQualifiedName().hashCode(); - } - - @Override - public String toString() { - return getQualifiedName(); - } - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java deleted file mode 100644 index d00f77d8dfcac7a42eb979f93cbda199afe1abf4..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import org.apache.hadoop.hive.serde.serdeConstants; - -public class DecimalTypeInfo extends PrimitiveTypeInfo { - private static final long serialVersionUID = 1L; - - private int precision; - private int scale; - - // no-arg constructor to make kyro happy. - public DecimalTypeInfo() { - super(serdeConstants.DECIMAL_TYPE_NAME); - } - - public DecimalTypeInfo(int precision, int scale) { - super(serdeConstants.DECIMAL_TYPE_NAME); - HiveDecimalUtils.validateParameter(precision, scale); - this.precision = precision; - this.scale = scale; - } - - @Override - public String getTypeName() { - return getQualifiedName(); - } - - @Override - public void setTypeName(String typeName) { - // No need to set type name, it should always be decimal - return; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (other == null || getClass() != other.getClass()) { - return false; - } - - DecimalTypeInfo dti = (DecimalTypeInfo)other; - - return this.precision() == dti.precision() && this.scale() == dti.scale(); - - } - - /** - * Generate the hashCode for this TypeInfo. - */ - @Override - public int hashCode() { - return 31 * (17 + precision) + scale; - } - - @Override - public String toString() { - return getQualifiedName(); - } - - @Override - public String getQualifiedName() { - return getQualifiedName(precision, scale); - } - - public static String getQualifiedName(int precision, int scale) { - StringBuilder sb = new StringBuilder(serdeConstants.DECIMAL_TYPE_NAME); - sb.append("("); - sb.append(precision); - sb.append(","); - sb.append(scale); - sb.append(")"); - return sb.toString(); - } - - public int precision() { - return precision; - } - - public int scale() { - return scale; - } - - @Override - public boolean accept(TypeInfo other) { - if (other == null || getClass() != other.getClass()) { - return false; - } - - DecimalTypeInfo dti = (DecimalTypeInfo)other; - // Make sure "this" has enough integer room to accomodate other's integer digits. - return this.precision() - this.scale() >= dti.precision() - dti.scale(); - } - - public int getPrecision() { - return precision; - } - - public void setPrecision(int precision) { - this.precision = precision; - } - - public int getScale() { - return scale; - } - - public void setScale(int scale) { - this.scale = scale; - } - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java index cee9c45d4b5eb5786e562a5cb59a34a441c5afd9..5c1722cbb8c2ae40cde36775d86bd34d34e7f85b 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java @@ -39,22 +39,6 @@ public static HiveDecimalWritable enforcePrecisionScale(HiveDecimalWritable writ return (result.isSet() ? result : null); } - public static void validateParameter(int precision, int scale) { - if (precision < 1 || precision > HiveDecimal.MAX_PRECISION) { - throw new IllegalArgumentException("Decimal precision out of allowed range [1," + - HiveDecimal.MAX_PRECISION + "]"); - } - - if (scale < 0 || scale > HiveDecimal.MAX_SCALE) { - throw new IllegalArgumentException("Decimal scale out of allowed range [0," + - HiveDecimal.MAX_SCALE + "]"); - } - - if (precision < scale) { - throw new IllegalArgumentException("Decimal scale must be less than or equal to precision"); - } - } - /** * Need to keep consistent with JdbcColumn.columnPrecision * diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java deleted file mode 100644 index d60a96bf95a7670e8fd7362c6fea70613331c6ea..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * A List Type has homogeneous elements. All elements of the List has the same - * TypeInfo which is returned by getListElementTypeInfo. - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public final class ListTypeInfo extends TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - private TypeInfo listElementTypeInfo; - - /** - * For java serialization use only. - */ - public ListTypeInfo() { - } - - @Override - public String getTypeName() { - return org.apache.hadoop.hive.serde.serdeConstants.LIST_TYPE_NAME + "<" - + listElementTypeInfo.getTypeName() + ">"; - } - - /** - * For java serialization use only. - */ - public void setListElementTypeInfo(TypeInfo listElementTypeInfo) { - this.listElementTypeInfo = listElementTypeInfo; - } - - /** - * For TypeInfoFactory use only. - */ - ListTypeInfo(TypeInfo elementTypeInfo) { - listElementTypeInfo = elementTypeInfo; - } - - @Override - public Category getCategory() { - return Category.LIST; - } - - public TypeInfo getListElementTypeInfo() { - return listElementTypeInfo; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof ListTypeInfo)) { - return false; - } - return getListElementTypeInfo().equals( - ((ListTypeInfo) other).getListElementTypeInfo()); - } - - @Override - public int hashCode() { - return listElementTypeInfo.hashCode(); - } - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java deleted file mode 100644 index 5ecae5899dcb9f929bd84874cf99d7cfe012d6da..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * A Map Type has homogeneous keys and homogeneous values. All keys of the Map - * have the same TypeInfo, which is returned by getMapKeyTypeInfo(); and all - * values of the Map has the same TypeInfo, which is returned by - * getMapValueTypeInfo(). - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public final class MapTypeInfo extends TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - - private TypeInfo mapKeyTypeInfo; - private TypeInfo mapValueTypeInfo; - - /** - * For java serialization use only. - */ - public MapTypeInfo() { - } - - @Override - public String getTypeName() { - return org.apache.hadoop.hive.serde.serdeConstants.MAP_TYPE_NAME + "<" - + mapKeyTypeInfo.getTypeName() + "," + mapValueTypeInfo.getTypeName() - + ">"; - } - - /** - * For java serialization use only. - */ - public void setMapKeyTypeInfo(TypeInfo mapKeyTypeInfo) { - this.mapKeyTypeInfo = mapKeyTypeInfo; - } - - /** - * For java serialization use only. - */ - public void setMapValueTypeInfo(TypeInfo mapValueTypeInfo) { - this.mapValueTypeInfo = mapValueTypeInfo; - } - - // For TypeInfoFactory use only - MapTypeInfo(TypeInfo keyTypeInfo, TypeInfo valueTypeInfo) { - mapKeyTypeInfo = keyTypeInfo; - mapValueTypeInfo = valueTypeInfo; - } - - @Override - public Category getCategory() { - return Category.MAP; - } - - public TypeInfo getMapKeyTypeInfo() { - return mapKeyTypeInfo; - } - - public TypeInfo getMapValueTypeInfo() { - return mapValueTypeInfo; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof MapTypeInfo)) { - return false; - } - MapTypeInfo o = (MapTypeInfo) other; - return o.getMapKeyTypeInfo().equals(getMapKeyTypeInfo()) - && o.getMapValueTypeInfo().equals(getMapValueTypeInfo()); - } - - @Override - public int hashCode() { - return mapKeyTypeInfo.hashCode() ^ mapValueTypeInfo.hashCode(); - } - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java deleted file mode 100644 index 98524adf0c18c57b0ae3e4d6e6ed685f66f05678..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; - -/** - * There are limited number of Primitive Types. All Primitive Types are defined - * by TypeInfoFactory.isPrimitiveClass(). - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public class PrimitiveTypeInfo extends TypeInfo implements Serializable { - private static final long serialVersionUID = 1L; - - // Base name (varchar vs fully qualified name such as varchar(200)). - protected String typeName; - - /** - * For java serialization use only. - */ - public PrimitiveTypeInfo() { - } - - /** - * For TypeInfoFactory use only. - */ - PrimitiveTypeInfo(String typeName) { - this.typeName = typeName; - } - - /** - * Returns the category of this TypeInfo. - */ - @Override - public Category getCategory() { - return Category.PRIMITIVE; - } - - public PrimitiveCategory getPrimitiveCategory() { - return getPrimitiveTypeEntry().primitiveCategory; - } - - public Class getPrimitiveWritableClass() { - return getPrimitiveTypeEntry().primitiveWritableClass; - } - - public Class getPrimitiveJavaClass() { - return getPrimitiveTypeEntry().primitiveJavaClass; - } - - // The following 2 methods are for java serialization use only. - public void setTypeName(String typeName) { - this.typeName = typeName; - } - - @Override - public String getTypeName() { - return typeName; - } - - public PrimitiveTypeEntry getPrimitiveTypeEntry() { - return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName); - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (other == null || getClass() != other.getClass()) { - return false; - } - - PrimitiveTypeInfo pti = (PrimitiveTypeInfo) other; - - return this.typeName.equals(pti.typeName); - } - - /** - * Generate the hashCode for this TypeInfo. - */ - @Override - public int hashCode() { - return typeName.hashCode(); - } - - @Override - public String toString() { - return typeName; - } -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java deleted file mode 100644 index 3984426734e0795f59f5a30c90a1f1556a60be88..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * StructTypeInfo represents the TypeInfo of a struct. A struct contains one or - * more fields each of which has a unique name and its own TypeInfo. Different - * fields can have the same or different TypeInfo. - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public final class StructTypeInfo extends TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - - private ArrayList allStructFieldNames; - private ArrayList allStructFieldTypeInfos; - - /** - * For java serialization use only. - */ - public StructTypeInfo() { - } - - @Override - public String getTypeName() { - StringBuilder sb = new StringBuilder(); - sb.append(serdeConstants.STRUCT_TYPE_NAME + "<"); - for (int i = 0; i < allStructFieldNames.size(); i++) { - if (i > 0) { - sb.append(","); - } - sb.append(allStructFieldNames.get(i)); - sb.append(":"); - sb.append(allStructFieldTypeInfos.get(i).getTypeName()); - } - sb.append(">"); - return sb.toString(); - } - - /** - * For java serialization use only. - */ - public void setAllStructFieldNames(ArrayList allStructFieldNames) { - this.allStructFieldNames = allStructFieldNames; - } - - /** - * For java serialization use only. - */ - public void setAllStructFieldTypeInfos( - ArrayList allStructFieldTypeInfos) { - this.allStructFieldTypeInfos = allStructFieldTypeInfos; - } - - /** - * For TypeInfoFactory use only. - */ - StructTypeInfo(List names, List typeInfos) { - allStructFieldNames = new ArrayList(names); - allStructFieldTypeInfos = new ArrayList(typeInfos); - } - - @Override - public Category getCategory() { - return Category.STRUCT; - } - - public ArrayList getAllStructFieldNames() { - return allStructFieldNames; - } - - public ArrayList getAllStructFieldTypeInfos() { - return allStructFieldTypeInfos; - } - - public TypeInfo getStructFieldTypeInfo(String field) { - String fieldLowerCase = field.toLowerCase(); - for (int i = 0; i < allStructFieldNames.size(); i++) { - if (fieldLowerCase.equalsIgnoreCase(allStructFieldNames.get(i))) { - return allStructFieldTypeInfos.get(i); - } - } - throw new RuntimeException("cannot find field " + field - + "(lowercase form: " + fieldLowerCase + ") in " + allStructFieldNames); - // return null; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof StructTypeInfo)) { - return false; - } - StructTypeInfo o = (StructTypeInfo) other; - Iterator namesIterator = getAllStructFieldNames().iterator(); - Iterator otherNamesIterator = o.getAllStructFieldNames().iterator(); - - // Compare the field names using ignore-case semantics - while (namesIterator.hasNext() && otherNamesIterator.hasNext()) { - if (!namesIterator.next().equalsIgnoreCase(otherNamesIterator.next())) { - return false; - } - } - - // Different number of field names - if (namesIterator.hasNext() || otherNamesIterator.hasNext()) { - return false; - } - - // Compare the field types - return o.getAllStructFieldTypeInfos().equals(getAllStructFieldTypeInfos()); - } - - @Override - public int hashCode() { - return allStructFieldNames.hashCode() ^ allStructFieldTypeInfos.hashCode(); - } - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java deleted file mode 100644 index e19d7b09cc5fe8cf8ab5605dbe7e422caa7457eb..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.time.ZoneId; -import java.util.Objects; - -import org.apache.hadoop.hive.common.type.TimestampTZUtil; -import org.apache.hadoop.hive.serde.serdeConstants; - -public class TimestampLocalTZTypeInfo extends PrimitiveTypeInfo { - private static final long serialVersionUID = 1L; - - private ZoneId timeZone; - - public TimestampLocalTZTypeInfo() { - super(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME); - } - - public TimestampLocalTZTypeInfo(String timeZoneStr) { - super(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME); - this.timeZone = TimestampTZUtil.parseTimeZone(timeZoneStr); - } - - @Override - public String getTypeName() { - return serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME; - } - - @Override - public void setTypeName(String typeName) { - // No need to set type name, it should always be timestamplocaltz - return; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (other == null || getClass() != other.getClass()) { - return false; - } - - TimestampLocalTZTypeInfo dti = (TimestampLocalTZTypeInfo) other; - - return this.timeZone().equals(dti.timeZone()); - } - - /** - * Generate the hashCode for this TypeInfo. - */ - @Override - public int hashCode() { - return Objects.hash(typeName, timeZone); - } - - @Override - public String toString() { - return getQualifiedName(); - } - - @Override - public String getQualifiedName() { - return getQualifiedName(timeZone); - } - - public static String getQualifiedName(ZoneId timeZone) { - StringBuilder sb = new StringBuilder(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME); - sb.append("('"); - sb.append(timeZone); - sb.append("')"); - return sb.toString(); - } - - public ZoneId timeZone() { - return timeZone; - } - - public ZoneId getTimeZone() { - return timeZone; - } - - public void setTimeZone(ZoneId timeZone) { - this.timeZone = timeZone; - } - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java deleted file mode 100644 index defb052e94a1202d88584484476f83a5a31d1152..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * Stores information about a type. Always use the TypeInfoFactory to create new - * TypeInfo objects. - * - * We support 8 categories of types: - * 1. Primitive objects (String, Number, etc) - * 2. List objects (a list of objects of a single type) - * 3. Map objects (a map from objects of one type to objects of another type) - * 4. Struct objects (a list of fields with names and their own types) - * 5. Union objects - * 6. Decimal objects - * 7. Char objects - * 8. Varchar objects - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public abstract class TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - - protected TypeInfo() { - } - - /** - * The Category of this TypeInfo. Possible values are Primitive, List, Map, - * Struct and Union, which corresponds to the 5 sub-classes of TypeInfo. - */ - public abstract Category getCategory(); - - /** - * A String representation of the TypeInfo. - */ - public abstract String getTypeName(); - - /** - * String representing the qualified type name. - * Qualified types should override this method. - * @return - */ - public String getQualifiedName() { - return getTypeName(); - } - - @Override - public String toString() { - return getTypeName(); - } - - @Override - public abstract boolean equals(Object o); - - @Override - public abstract int hashCode(); - - public boolean accept(TypeInfo other) { - return this.equals(other); - } - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java deleted file mode 100644 index 66f5f8fec69c0b66c191351b41adc3bd6d222f58..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java +++ /dev/null @@ -1,279 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.time.ZoneId; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.hadoop.hive.common.type.HiveChar; -import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; - -/** - * TypeInfoFactory can be used to create the TypeInfo object for any types. - * - * TypeInfo objects are all read-only so we can reuse them easily. - * TypeInfoFactory has internal cache to make sure we don't create 2 TypeInfo - * objects that represents the same type. - */ -public final class TypeInfoFactory { - - private TypeInfoFactory() { - // prevent instantiation - } - - public static final PrimitiveTypeInfo voidTypeInfo = new PrimitiveTypeInfo(serdeConstants.VOID_TYPE_NAME); - public static final PrimitiveTypeInfo booleanTypeInfo = new PrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME); - public static final PrimitiveTypeInfo intTypeInfo = new PrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME); - public static final PrimitiveTypeInfo longTypeInfo = new PrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME); - public static final PrimitiveTypeInfo stringTypeInfo = new PrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); - public static final PrimitiveTypeInfo charTypeInfo = new CharTypeInfo(HiveChar.MAX_CHAR_LENGTH); - public static final PrimitiveTypeInfo varcharTypeInfo = new VarcharTypeInfo(HiveVarchar.MAX_VARCHAR_LENGTH); - public static final PrimitiveTypeInfo floatTypeInfo = new PrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME); - public static final PrimitiveTypeInfo doubleTypeInfo = new PrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME); - public static final PrimitiveTypeInfo byteTypeInfo = new PrimitiveTypeInfo(serdeConstants.TINYINT_TYPE_NAME); - public static final PrimitiveTypeInfo shortTypeInfo = new PrimitiveTypeInfo(serdeConstants.SMALLINT_TYPE_NAME); - public static final PrimitiveTypeInfo dateTypeInfo = new PrimitiveTypeInfo(serdeConstants.DATE_TYPE_NAME); - public static final PrimitiveTypeInfo timestampTypeInfo = new PrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME); - public static final PrimitiveTypeInfo intervalYearMonthTypeInfo = new PrimitiveTypeInfo(serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME); - public static final PrimitiveTypeInfo intervalDayTimeTypeInfo = new PrimitiveTypeInfo(serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME); - public static final PrimitiveTypeInfo binaryTypeInfo = new PrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME); - - /** - * A DecimalTypeInfo instance that has max precision and max scale. - */ - public static final DecimalTypeInfo decimalTypeInfo = new DecimalTypeInfo(HiveDecimal.SYSTEM_DEFAULT_PRECISION, - HiveDecimal.SYSTEM_DEFAULT_SCALE); - - /** - * A TimestampTZTypeInfo with system default time zone. - */ - public static final TimestampLocalTZTypeInfo timestampLocalTZTypeInfo = new TimestampLocalTZTypeInfo( - ZoneId.systemDefault().getId()); - - public static final PrimitiveTypeInfo unknownTypeInfo = new PrimitiveTypeInfo("unknown"); - - // Map from type name (such as int or varchar(40) to the corresponding PrimitiveTypeInfo - // instance. - private static ConcurrentHashMap cachedPrimitiveTypeInfo = - new ConcurrentHashMap(); - static { - cachedPrimitiveTypeInfo.put(serdeConstants.VOID_TYPE_NAME, voidTypeInfo); - cachedPrimitiveTypeInfo.put(serdeConstants.BOOLEAN_TYPE_NAME, booleanTypeInfo); - cachedPrimitiveTypeInfo.put(serdeConstants.INT_TYPE_NAME, intTypeInfo); - cachedPrimitiveTypeInfo.put(serdeConstants.BIGINT_TYPE_NAME, longTypeInfo); - cachedPrimitiveTypeInfo.put(serdeConstants.STRING_TYPE_NAME, stringTypeInfo); - cachedPrimitiveTypeInfo.put(charTypeInfo.getQualifiedName(), charTypeInfo); - cachedPrimitiveTypeInfo.put(varcharTypeInfo.getQualifiedName(), varcharTypeInfo); - cachedPrimitiveTypeInfo.put(serdeConstants.FLOAT_TYPE_NAME, floatTypeInfo); - cachedPrimitiveTypeInfo.put(serdeConstants.DOUBLE_TYPE_NAME, doubleTypeInfo); - cachedPrimitiveTypeInfo.put(serdeConstants.TINYINT_TYPE_NAME, byteTypeInfo); - cachedPrimitiveTypeInfo.put(serdeConstants.SMALLINT_TYPE_NAME, shortTypeInfo); - cachedPrimitiveTypeInfo.put(serdeConstants.DATE_TYPE_NAME, dateTypeInfo); - cachedPrimitiveTypeInfo.put(serdeConstants.TIMESTAMP_TYPE_NAME, timestampTypeInfo); - cachedPrimitiveTypeInfo.put(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME, timestampLocalTZTypeInfo); - cachedPrimitiveTypeInfo.put(serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME, intervalYearMonthTypeInfo); - cachedPrimitiveTypeInfo.put(serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME, intervalDayTimeTypeInfo); - cachedPrimitiveTypeInfo.put(serdeConstants.BINARY_TYPE_NAME, binaryTypeInfo); - cachedPrimitiveTypeInfo.put(decimalTypeInfo.getQualifiedName(), decimalTypeInfo); - cachedPrimitiveTypeInfo.put("unknown", unknownTypeInfo); - } - - /** - * Get PrimitiveTypeInfo instance for the given type name of a type - * including types with parameters, such as varchar(20). - * - * @param typeName type name possibly with parameters. - * @return aPrimitiveTypeInfo instance - */ - public static PrimitiveTypeInfo getPrimitiveTypeInfo(String typeName) { - PrimitiveTypeInfo result = cachedPrimitiveTypeInfo.get(typeName); - if (result != null) { - return result; - } - - // Not found in the cache. Must be parameterized types. Create it. - result = createPrimitiveTypeInfo(typeName); - if (result == null) { - throw new RuntimeException("Error creating PrimitiveTypeInfo instance for " + typeName); - } - - PrimitiveTypeInfo prev = cachedPrimitiveTypeInfo.putIfAbsent(typeName, result); - if (prev != null) { - result = prev; - } - return result; - } - - /** - * Create PrimitiveTypeInfo instance for the given full name of the type. The returned - * type is one of the parameterized type info such as VarcharTypeInfo. - * - * @param fullName Fully qualified name of the type - * @return PrimitiveTypeInfo instance - */ - private static PrimitiveTypeInfo createPrimitiveTypeInfo(String fullName) { - String baseName = TypeInfoUtils.getBaseName(fullName); - PrimitiveTypeEntry typeEntry = - PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(baseName); - if (null == typeEntry) { - throw new RuntimeException("Unknown type " + fullName); - } - - TypeInfoUtils.PrimitiveParts parts = TypeInfoUtils.parsePrimitiveParts(fullName); - if (parts.typeParams == null || parts.typeParams.length < 1) { - return null; - } - - switch (typeEntry.primitiveCategory) { - case CHAR: - if (parts.typeParams.length != 1) { - return null; - } - return new CharTypeInfo(Integer.valueOf(parts.typeParams[0])); - case VARCHAR: - if (parts.typeParams.length != 1) { - return null; - } - return new VarcharTypeInfo(Integer.valueOf(parts.typeParams[0])); - case DECIMAL: - if (parts.typeParams.length != 2) { - return null; - } - return new DecimalTypeInfo(Integer.valueOf(parts.typeParams[0]), - Integer.valueOf(parts.typeParams[1])); - case TIMESTAMPLOCALTZ: - if (parts.typeParams.length != 1) { - return null; - } - return new TimestampLocalTZTypeInfo(parts.typeParams[0]); - default: - return null; - } - } - - public static CharTypeInfo getCharTypeInfo(int length) { - String fullName = BaseCharTypeInfo.getQualifiedName(serdeConstants.CHAR_TYPE_NAME, length); - return (CharTypeInfo) getPrimitiveTypeInfo(fullName); - } - - public static VarcharTypeInfo getVarcharTypeInfo(int length) { - String fullName = BaseCharTypeInfo.getQualifiedName(serdeConstants.VARCHAR_TYPE_NAME, length); - return (VarcharTypeInfo) getPrimitiveTypeInfo(fullName); - } - - public static DecimalTypeInfo getDecimalTypeInfo(int precision, int scale) { - String fullName = DecimalTypeInfo.getQualifiedName(precision, scale); - return (DecimalTypeInfo) getPrimitiveTypeInfo(fullName); - }; - - public static TimestampLocalTZTypeInfo getTimestampTZTypeInfo(ZoneId defaultTimeZone) { - String fullName = TimestampLocalTZTypeInfo.getQualifiedName(defaultTimeZone); - return (TimestampLocalTZTypeInfo) getPrimitiveTypeInfo(fullName); - }; - - public static TypeInfo getPrimitiveTypeInfoFromPrimitiveWritable( - Class clazz) { - String typeName = PrimitiveObjectInspectorUtils - .getTypeNameFromPrimitiveWritable(clazz); - if (typeName == null) { - throw new RuntimeException("Internal error: Cannot get typeName for " - + clazz); - } - return getPrimitiveTypeInfo(typeName); - } - - public static TypeInfo getPrimitiveTypeInfoFromJavaPrimitive(Class clazz) { - return getPrimitiveTypeInfo(PrimitiveObjectInspectorUtils - .getTypeNameFromPrimitiveJava(clazz)); - } - - static ConcurrentHashMap>, TypeInfo> cachedStructTypeInfo = - new ConcurrentHashMap>, TypeInfo>(); - - public static TypeInfo getStructTypeInfo(List names, - List typeInfos) { - ArrayList> signature = new ArrayList>(2); - signature.add(names); - signature.add(typeInfos); - TypeInfo result = cachedStructTypeInfo.get(signature); - if (result == null) { - result = new StructTypeInfo(names, typeInfos); - TypeInfo prev = cachedStructTypeInfo.putIfAbsent(signature, result); - if (prev != null) { - result = prev; - } - } - return result; - } - - static ConcurrentHashMap, TypeInfo> cachedUnionTypeInfo = - new ConcurrentHashMap, TypeInfo>(); - - public static TypeInfo getUnionTypeInfo(List typeInfos) { - TypeInfo result = cachedUnionTypeInfo.get(typeInfos); - if (result == null) { - result = new UnionTypeInfo(typeInfos); - TypeInfo prev = cachedUnionTypeInfo.putIfAbsent(typeInfos, result); - if (prev != null) { - result = prev; - } - } - return result; - } - - static ConcurrentHashMap cachedListTypeInfo = new ConcurrentHashMap(); - - public static TypeInfo getListTypeInfo(TypeInfo elementTypeInfo) { - TypeInfo result = cachedListTypeInfo.get(elementTypeInfo); - if (result == null) { - result = new ListTypeInfo(elementTypeInfo); - TypeInfo prev = cachedListTypeInfo.putIfAbsent(elementTypeInfo, result); - if (prev != null) { - result = prev; - } - } - return result; - } - - static ConcurrentHashMap, TypeInfo> cachedMapTypeInfo = - new ConcurrentHashMap, TypeInfo>(); - - public static TypeInfo getMapTypeInfo(TypeInfo keyTypeInfo, - TypeInfo valueTypeInfo) { - ArrayList signature = new ArrayList(2); - signature.add(keyTypeInfo); - signature.add(valueTypeInfo); - TypeInfo result = cachedMapTypeInfo.get(signature); - if (result == null) { - result = new MapTypeInfo(keyTypeInfo, valueTypeInfo); - TypeInfo prev = cachedMapTypeInfo.putIfAbsent(signature, result); - if (prev != null) { - result = prev; - } - } - return result; - } - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java index 9b85c65c427d79fbd9d71bb79579029f2dfb4910..a82c15753964028f38d645d3757bc848d0382cc1 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java @@ -25,14 +25,11 @@ import java.util.ArrayList; import java.util.EnumMap; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -47,7 +44,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -265,332 +261,6 @@ public static boolean doPrimitiveCategoriesMatch(TypeInfo ti1, TypeInfo ti2) { return false; } - /** - * Parse a recursive TypeInfo list String. For example, the following inputs - * are valid inputs: - * "int,string,map,list>>,list>" - * The separators between TypeInfos can be ",", ":", or ";". - * - * In order to use this class: TypeInfoParser parser = new - * TypeInfoParser("int,string"); ArrayList typeInfos = - * parser.parseTypeInfos(); - */ - private static class TypeInfoParser { - - private static class Token { - public int position; - public String text; - public boolean isType; - - @Override - public String toString() { - return "" + position + ":" + text; - } - }; - - private static boolean isTypeChar(char c) { - return Character.isLetterOrDigit(c) || c == '_' || c == '.' || c == ' ' || c == '$'; - } - - /** - * Tokenize the typeInfoString. The rule is simple: all consecutive - * alphadigits and '_', '.' are in one token, and all other characters are - * one character per token. - * - * tokenize("map") should return - * ["map","<","int",",","string",">"] - * - * Note that we add '$' in new Calcite return path. As '$' will not appear - * in any type in Hive, it is safe to do so. - */ - private static ArrayList tokenize(String typeInfoString) { - ArrayList tokens = new ArrayList(0); - int begin = 0; - int end = 1; - while (end <= typeInfoString.length()) { - // last character ends a token? - // if there are quotes, all the text between the quotes - // is considered a single token (this can happen for - // timestamp with local time-zone) - if (begin > 0 && - typeInfoString.charAt(begin - 1) == '(' && - typeInfoString.charAt(begin) == '\'') { - // Ignore starting quote - begin++; - do { - end++; - } while (typeInfoString.charAt(end) != '\''); - } else if (typeInfoString.charAt(begin) == '\'' && - typeInfoString.charAt(begin + 1) == ')') { - // Ignore closing quote - begin++; - end++; - } - if (end == typeInfoString.length() - || !isTypeChar(typeInfoString.charAt(end - 1)) - || !isTypeChar(typeInfoString.charAt(end))) { - Token t = new Token(); - t.position = begin; - t.text = typeInfoString.substring(begin, end); - t.isType = isTypeChar(typeInfoString.charAt(begin)); - tokens.add(t); - begin = end; - } - end++; - } - return tokens; - } - - public TypeInfoParser(String typeInfoString) { - this.typeInfoString = typeInfoString; - typeInfoTokens = tokenize(typeInfoString); - } - - private final String typeInfoString; - private final ArrayList typeInfoTokens; - private ArrayList typeInfos; - private int iToken; - - public ArrayList parseTypeInfos() { - typeInfos = new ArrayList(); - iToken = 0; - while (iToken < typeInfoTokens.size()) { - typeInfos.add(parseType()); - if (iToken < typeInfoTokens.size()) { - Token separator = typeInfoTokens.get(iToken); - if (",".equals(separator.text) || ";".equals(separator.text) - || ":".equals(separator.text)) { - iToken++; - } else { - throw new IllegalArgumentException( - "Error: ',', ':', or ';' expected at position " - + separator.position + " from '" + typeInfoString + "' " - + typeInfoTokens); - } - } - } - return typeInfos; - } - - private Token peek() { - if (iToken < typeInfoTokens.size()) { - return typeInfoTokens.get(iToken); - } else { - return null; - } - } - - private Token expect(String item) { - return expect(item, null); - } - - private Token expect(String item, String alternative) { - if (iToken >= typeInfoTokens.size()) { - throw new IllegalArgumentException("Error: " + item - + " expected at the end of '" + typeInfoString + "'"); - } - Token t = typeInfoTokens.get(iToken); - if (item.equals("type")) { - if (!serdeConstants.LIST_TYPE_NAME.equals(t.text) - && !serdeConstants.MAP_TYPE_NAME.equals(t.text) - && !serdeConstants.STRUCT_TYPE_NAME.equals(t.text) - && !serdeConstants.UNION_TYPE_NAME.equals(t.text) - && null == PrimitiveObjectInspectorUtils - .getTypeEntryFromTypeName(t.text) - && !t.text.equals(alternative)) { - throw new IllegalArgumentException("Error: " + item - + " expected at the position " + t.position + " of '" - + typeInfoString + "' but '" + t.text + "' is found."); - } - } else if (item.equals("name")) { - if (!t.isType && !t.text.equals(alternative)) { - throw new IllegalArgumentException("Error: " + item - + " expected at the position " + t.position + " of '" - + typeInfoString + "' but '" + t.text + "' is found."); - } - } else { - if (!item.equals(t.text) && !t.text.equals(alternative)) { - throw new IllegalArgumentException("Error: " + item - + " expected at the position " + t.position + " of '" - + typeInfoString + "' but '" + t.text + "' is found."); - } - } - iToken++; - return t; - } - - private String[] parseParams() { - List params = new LinkedList(); - - Token t = peek(); - if (t != null && t.text.equals("(")) { - expect("("); - - // checking for null in the for-loop condition prevents null-ptr exception - // and allows us to fail more gracefully with a parsing error. - for(t = peek(); (t == null) || !t.text.equals(")"); t = expect(",",")")) { - params.add(expect("name").text); - } - if (params.size() == 0) { - throw new IllegalArgumentException( - "type parameters expected for type string " + typeInfoString); - } - } - - return params.toArray(new String[params.size()]); - } - - private TypeInfo parseType() { - - Token t = expect("type"); - - // Is this a primitive type? - PrimitiveTypeEntry typeEntry = - PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(t.text); - if (typeEntry != null && typeEntry.primitiveCategory != PrimitiveCategory.UNKNOWN ) { - String[] params = parseParams(); - switch (typeEntry.primitiveCategory) { - case CHAR: - case VARCHAR: - if (params == null || params.length == 0) { - throw new IllegalArgumentException(typeEntry.typeName - + " type is specified without length: " + typeInfoString); - } - - int length = 1; - if (params.length == 1) { - length = Integer.parseInt(params[0]); - if (typeEntry.primitiveCategory == PrimitiveCategory.VARCHAR) { - BaseCharUtils.validateVarcharParameter(length); - return TypeInfoFactory.getVarcharTypeInfo(length); - } else { - BaseCharUtils.validateCharParameter(length); - return TypeInfoFactory.getCharTypeInfo(length); - } - } else if (params.length > 1) { - throw new IllegalArgumentException( - "Type " + typeEntry.typeName+ " only takes one parameter, but " + - params.length + " is seen"); - } - - case DECIMAL: - int precision = HiveDecimal.USER_DEFAULT_PRECISION; - int scale = HiveDecimal.USER_DEFAULT_SCALE; - if (params == null || params.length == 0) { - // It's possible that old metadata still refers to "decimal" as a column type w/o - // precision/scale. In this case, the default (10,0) is assumed. Thus, do nothing here. - } else if (params.length == 1) { - // only precision is specified - precision = Integer.valueOf(params[0]); - HiveDecimalUtils.validateParameter(precision, scale); - } else if (params.length == 2) { - // New metadata always have two parameters. - precision = Integer.parseInt(params[0]); - scale = Integer.parseInt(params[1]); - HiveDecimalUtils.validateParameter(precision, scale); - } else if (params.length > 2) { - throw new IllegalArgumentException("Type decimal only takes two parameter, but " + - params.length + " is seen"); - } - return TypeInfoFactory.getDecimalTypeInfo(precision, scale); - - default: - return TypeInfoFactory.getPrimitiveTypeInfo(typeEntry.typeName); - } - } - - // Is this a list type? - if (serdeConstants.LIST_TYPE_NAME.equals(t.text)) { - expect("<"); - TypeInfo listElementType = parseType(); - expect(">"); - return TypeInfoFactory.getListTypeInfo(listElementType); - } - - // Is this a map type? - if (serdeConstants.MAP_TYPE_NAME.equals(t.text)) { - expect("<"); - TypeInfo mapKeyType = parseType(); - expect(","); - TypeInfo mapValueType = parseType(); - expect(">"); - return TypeInfoFactory.getMapTypeInfo(mapKeyType, mapValueType); - } - - // Is this a struct type? - if (serdeConstants.STRUCT_TYPE_NAME.equals(t.text)) { - ArrayList fieldNames = new ArrayList(); - ArrayList fieldTypeInfos = new ArrayList(); - boolean first = true; - do { - if (first) { - expect("<"); - first = false; - } else { - Token separator = expect(">", ","); - if (separator.text.equals(">")) { - // end of struct - break; - } - } - Token name = expect("name",">"); - if (name.text.equals(">")) { - break; - } - fieldNames.add(name.text); - expect(":"); - fieldTypeInfos.add(parseType()); - } while (true); - - return TypeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos); - } - // Is this a union type? - if (serdeConstants.UNION_TYPE_NAME.equals(t.text)) { - List objectTypeInfos = new ArrayList(); - boolean first = true; - do { - if (first) { - expect("<"); - first = false; - } else { - Token separator = expect(">", ","); - if (separator.text.equals(">")) { - // end of union - break; - } - } - objectTypeInfos.add(parseType()); - } while (true); - - return TypeInfoFactory.getUnionTypeInfo(objectTypeInfos); - } - - throw new RuntimeException("Internal error parsing position " - + t.position + " of '" + typeInfoString + "'"); - } - - public PrimitiveParts parsePrimitiveParts() { - PrimitiveParts parts = new PrimitiveParts(); - Token t = expect("type"); - parts.typeName = t.text; - parts.typeParams = parseParams(); - return parts; - } - } - - public static class PrimitiveParts { - public String typeName; - public String[] typeParams; - } - - /** - * Make some of the TypeInfo parsing available as a utility. - */ - public static PrimitiveParts parsePrimitiveParts(String typeInfoString) { - TypeInfoParser parser = new TypeInfoParser(typeInfoString); - return parser.parsePrimitiveParts(); - } - static ConcurrentHashMap cachedStandardObjectInspector = new ConcurrentHashMap(); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnionTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnionTypeInfo.java deleted file mode 100644 index 7157d80bdf093798690397e934d6b192725b2ced..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnionTypeInfo.java +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * UnionTypeInfo represents the TypeInfo of an union. A union holds only one - * field of the specified fields at any point of time. The fields, a Union can - * hold, can have the same or different TypeInfo. - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public class UnionTypeInfo extends TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - - private List allUnionObjectTypeInfos; - - /** - * For java serialization use only. - */ - public UnionTypeInfo() { - } - - @Override - public String getTypeName() { - StringBuilder sb = new StringBuilder(); - sb.append(serdeConstants.UNION_TYPE_NAME + "<"); - for (int i = 0; i < allUnionObjectTypeInfos.size(); i++) { - if (i > 0) { - sb.append(","); - } - sb.append(allUnionObjectTypeInfos.get(i).getTypeName()); - } - sb.append(">"); - return sb.toString(); - } - - /** - * For java serialization use only. - */ - public void setAllUnionObjectTypeInfos( - List allUnionObjectTypeInfos) { - this.allUnionObjectTypeInfos = allUnionObjectTypeInfos; - } - - /** - * For TypeInfoFactory use only. - */ - UnionTypeInfo(List typeInfos) { - allUnionObjectTypeInfos = new ArrayList(); - allUnionObjectTypeInfos.addAll(typeInfos); - } - - @Override - public Category getCategory() { - return Category.UNION; - } - - public List getAllUnionObjectTypeInfos() { - return allUnionObjectTypeInfos; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof UnionTypeInfo)) { - return false; - } - UnionTypeInfo o = (UnionTypeInfo) other; - - // Compare the field types - return o.getAllUnionObjectTypeInfos().equals(getAllUnionObjectTypeInfos()); - } - - @Override - public int hashCode() { - return allUnionObjectTypeInfos.hashCode(); - } -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java deleted file mode 100644 index cb5350398e7576d5c686c5cadc2f221021b9162f..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import org.apache.hadoop.hive.serde.serdeConstants; - -public class VarcharTypeInfo extends BaseCharTypeInfo { - private static final long serialVersionUID = 1L; - - // no-arg constructor to make kyro happy. - public VarcharTypeInfo() { - super(serdeConstants.VARCHAR_TYPE_NAME); - } - - public VarcharTypeInfo(int length) { - super(serdeConstants.VARCHAR_TYPE_NAME, length); - BaseCharUtils.validateVarcharParameter(length); - } - - @Override - public String getTypeName() { - return getQualifiedName(); - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (other == null || getClass() != other.getClass()) { - return false; - } - - VarcharTypeInfo pti = (VarcharTypeInfo) other; - - return this.getLength() == pti.getLength(); - } - - /** - * Generate the hashCode for this TypeInfo. - */ - @Override - public int hashCode() { - return getLength(); - } - - @Override - public String toString() { - return getQualifiedName(); - } - -} diff --git a/service/pom.xml b/service/pom.xml index 9ad755537d4e9c2848626138878371b43fbd655d..555cee81ec8079d371126e8efad85b6bff92c400 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -65,6 +65,11 @@ + org.apache.hive + hive-storage-api + ${storage-api.version} + + javax.servlet.jsp javax.servlet.jsp-api ${javax-servlet-jsp.version} diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index 85c3222640f8bc78781d944bca66ec67472468c0..bdded94f2bb3562263e533a1d1a42f310d5ae623 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -229,13 +229,18 @@ - + org.apache.hive hive-storage-api ${storage-api.version} + org.apache.hive + hive-serde-api + ${storage-api.version} + + org.apache.logging.log4j log4j-slf4j-impl ${log4j2.version} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java index d5dea4dc3ca6a83a863326e1c75e2480898d00af..124d6508ce0a44661d1fe7abc7206ee129b617fa 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java @@ -224,7 +224,7 @@ public static boolean areColTypesCompatible(String from, String to) { if (from.equals(VOID_TYPE_NAME)) return true; // Allow date to string casts. NOTE: I suspect this is the reverse of what we actually - // want, but it matches the code in o.a.h.h.serde2.typeinfo.TypeInfoUtils. I can't see how + // want, but it matches the code in o.a.h.h.serde2.typeinfo.StorageSchemaUtils. I can't see how // users would be altering date columns into string columns. The other I easily see since // Hive did not originally support datetime types. Also, the comment in the Hive code // says string to date, even though the code does the opposite. But for now I'm keeping @@ -248,6 +248,8 @@ public static boolean areColTypesCompatible(String from, String to) { public static final char COLUMN_COMMENTS_DELIMITER = '\0'; + public static final String LIST_COLUMN_COMMENTS = "columns.comments"; + private static HashMap typeToThriftTypeMap; static { typeToThriftTypeMap = new HashMap<>(); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultStorageSchemaReader.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultStorageSchemaReader.java deleted file mode 100644 index 1dbfa4272cd5368242d335fbde564d35cac26bba..0000000000000000000000000000000000000000 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultStorageSchemaReader.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.api.EnvironmentContext; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Table; - -import java.util.List; - -/** - * Default StorageSchemaReader. This just throws as the metastore currently doesn't know how to - * read schemas from storage. - */ -public class DefaultStorageSchemaReader implements StorageSchemaReader { - @Override - public List readSchema(Table tbl, EnvironmentContext envContext, - Configuration conf) throws MetaException { - throw new UnsupportedOperationException("Storage schema reading not supported"); - } -} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 69d26c4fee9fa0301d351c4aeee9e0b64a144b4e..2026a19e9ceca9e176dcd8043aec3caf429e277e 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -53,7 +53,6 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; import java.util.regex.Pattern; import javax.jdo.JDOException; @@ -132,6 +131,7 @@ import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; import org.apache.hadoop.hive.metastore.metrics.PerfLogger; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; +import org.apache.hadoop.hive.metastore.schema.reader.StorageSchemaReader; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager; import org.apache.hadoop.hive.metastore.security.TUGIContainingTransport; diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/StorageSchemaReader.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/StorageSchemaReader.java deleted file mode 100644 index 6251e23991b8f898f939f5848fc5cbf5e8ceb07c..0000000000000000000000000000000000000000 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/StorageSchemaReader.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.api.EnvironmentContext; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Table; - -import java.util.List; - -/** - * An interface to implement reading schemas from stored data. - */ -@InterfaceAudience.Public -@InterfaceStability.Evolving -interface StorageSchemaReader { - /** - * Read the schema from the storage representation of the table. - * @param tbl metastore table object - * @param envContext environment context - * @param conf current configuration file - * @return list of field schemas - * @throws MetaException if the table storage could not be read - */ - List readSchema(Table tbl, EnvironmentContext envContext, Configuration conf) - throws MetaException; -} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 57692d3e92ac820efa15fa54bea86e90dcfe5d93..0e4beb8236be3b2919d97614e07932b543910e46 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -19,7 +19,7 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader; +import org.apache.hadoop.hive.metastore.schema.reader.DefaultStorageSchemaReader; import org.apache.hadoop.hive.metastore.HiveAlterHandler; import org.apache.hadoop.hive.metastore.MetastoreTaskThread; import org.apache.hadoop.hive.metastore.events.EventCleanerTask; diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/DefaultStorageSchemaReader.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/DefaultStorageSchemaReader.java new file mode 100644 index 0000000000000000000000000000000000000000..31a7fcfa0c240bb1f448f911a6f1f9bc6c691cc4 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/DefaultStorageSchemaReader.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.schema.reader; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.ColumnType; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.schema.utils.AvroSchemaUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.schema.utils.StorageSchemaUtils; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.avro.AvroSerdeException; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import static org.apache.hadoop.hive.metastore.ColumnType.LIST_COLUMN_COMMENTS; + +/** + * Default StorageSchemaReader. This just throws as the metastore currently doesn't know how to + * read schemas from storage. + */ +public class DefaultStorageSchemaReader implements StorageSchemaReader { + private final static Logger LOG = LoggerFactory.getLogger(DefaultStorageSchemaReader.class); + + private static final String AVRO_SERIALIZATION_LIB = + "org.apache.hadoop.hive.serde2.avro.AvroSerDe"; + + @Override + public List readSchema(Table tbl, EnvironmentContext envContext, + Configuration conf) throws MetaException { + String serializationLib = tbl.getSd().getSerdeInfo().getSerializationLib(); + if (null == serializationLib || MetastoreConf + .getStringCollection(conf, MetastoreConf.ConfVars.SERDES_USING_METASTORE_FOR_SCHEMA) + .contains(serializationLib)) { + //safety check to make sure we should be using storage schema reader for this table + throw new MetaException( + "Invalid usage of default storage schema reader for table " + tbl.getTableName() + + " with storage descriptor " + tbl.getSd().getSerdeInfo().getSerializationLib()); + } + Properties tblMetadataProperties = MetaStoreUtils.getTableMetadata(tbl); + if(AVRO_SERIALIZATION_LIB.equals(serializationLib)) { + //in case of avro table use AvroStorageSchemaReader utils + try { + return AvroSchemaUtils.getFieldsFromAvroSchema(conf, tblMetadataProperties); + } catch (AvroSerdeException e) { + LOG.warn("Exception received while reading avro schema for table " + tbl.getTableName(), e); + throw new MetaException(e.getMessage()); + } catch (IOException e) { + LOG.warn("Exception received while reading avro schema for table " + tbl.getTableName(), e); + throw new MetaException(e.getMessage()); + } + } else { + return getFieldSchemasFromTableMetadata(tblMetadataProperties); + } + } + + /** + * This method implements a generic way to get the FieldSchemas from the table metadata + * properties like column names and column types. Most of the serdes have the same implemention + * in their initialize method + * //TODO refactor the common code from the serdes and move it to serde-api so that there is no + * //duplicate code + * + * @return list of FieldSchema objects + */ + public static List getFieldSchemasFromTableMetadata( + Properties tblMetadataProperties) { + List columnNames = null; + List columnTypes = null; + // Get column names and types + String columnNameProperty = tblMetadataProperties.getProperty(serdeConstants.LIST_COLUMNS); + String columnTypeProperty = tblMetadataProperties.getProperty(serdeConstants.LIST_COLUMN_TYPES); + final String columnNameDelimiter = tblMetadataProperties + .containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? tblMetadataProperties + .getProperty(serdeConstants.COLUMN_NAME_DELIMITER) : String + .valueOf(StorageSchemaUtils.COMMA); + // all table column names + if (columnNameProperty.isEmpty()) { + columnNames = Collections.emptyList(); + } else { + columnNames = Arrays.asList(columnNameProperty.split(columnNameDelimiter)); + } + + // all column types + if (columnTypeProperty.isEmpty()) { + columnTypes = Collections.emptyList(); + } else { + columnTypes = StorageSchemaUtils.getTypeInfosFromTypeString(columnTypeProperty); + } + + final String columnCommentProperty = + tblMetadataProperties.getProperty(LIST_COLUMN_COMMENTS, ""); + List columnComments = null; + if (columnCommentProperty == null || columnCommentProperty.isEmpty()) { + columnComments = new ArrayList<>(0); + } else { + columnComments = Arrays.asList( + columnCommentProperty.split(String.valueOf(ColumnType.COLUMN_COMMENTS_DELIMITER))); + } + LOG.debug("columns: {}, {}", columnNameProperty, columnNames); + LOG.debug("types: {}, {} ", columnTypeProperty, columnTypes); + LOG.debug("comments: {} ", columnCommentProperty); + return getFieldSchemaFromColumnInfo(columnNames, columnTypes, columnComments); + } + + private static List getFieldSchemaFromColumnInfo(List columnNames, + List columnTypes, List columnComments) { + int len = columnNames.size(); + List fieldSchemas = new ArrayList<>(len); + for (int i = 0; i < len; i++) { + FieldSchema fieldSchema = new FieldSchema(); + fieldSchema.setName(columnNames.get(i)); + //In case of complex types getTypeName() will recusively go into typeName + //of individual fields when the ColumnType was constructed + //in SchemaToTypeInfo.generateColumnTypes in the constructor + fieldSchema.setType(columnTypes.get(i).getTypeName()); + fieldSchema.setComment(StorageSchemaUtils.determineFieldComment(columnComments.get(i))); + } + return fieldSchemas; + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/SerDeStorageSchemaReader.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/SerDeStorageSchemaReader.java new file mode 100644 index 0000000000000000000000000000000000000000..fd09dd49bf0b4bd13bbff6d5b2d478b503257fc4 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/SerDeStorageSchemaReader.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.schema.reader; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.schema.utils.SerDeUtils; +import org.apache.hadoop.hive.metastore.utils.StringUtils; + +import java.util.List; + +/** + * In order to use this Storage schema reader you should add the hive-serde jar in the classpath + * of the metastore. + */ +public class SerDeStorageSchemaReader implements StorageSchemaReader { + @Override + public List readSchema(Table tbl, EnvironmentContext envContext, Configuration conf) + throws MetaException { + ClassLoader orgHiveLoader = null; + try { + if (envContext != null) { + String addedJars = envContext.getProperties().get("hive.added.jars.path"); + if (org.apache.commons.lang.StringUtils.isNotBlank(addedJars)) { + //for thread safe + orgHiveLoader = conf.getClassLoader(); + ClassLoader loader = org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.addToClassPath( + orgHiveLoader, org.apache.commons.lang.StringUtils.split(addedJars, ",")); + conf.setClassLoader(loader); + } + } + + Deserializer s = SerDeUtils.getDeserializer(conf, tbl, false); + return SerDeUtils.getFieldsFromDeserializer(tbl.getTableName(), s); + } catch (Exception e) { + StringUtils.stringifyException(e); + throw new MetaException(e.getMessage()); + } finally { + if (orgHiveLoader != null) { + conf.setClassLoader(orgHiveLoader); + } + } + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/StorageSchemaReader.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/StorageSchemaReader.java new file mode 100644 index 0000000000000000000000000000000000000000..539604ca8bcdc1c88fdd37691c6dcb09284edeb0 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/StorageSchemaReader.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.schema.reader; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; + +import java.util.List; + +/** + * An interface to implement reading schemas from stored data. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving public interface StorageSchemaReader { + /** + * Read the schema from the storage representation of the table. + * @param tbl metastore table object + * @param envContext environment context + * @param conf current configuration file + * @return list of field schemas + * @throws MetaException if the table storage could not be read + */ + List readSchema(Table tbl, EnvironmentContext envContext, Configuration conf) + throws MetaException; +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/AvroFieldSchemaGenerator.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/AvroFieldSchemaGenerator.java new file mode 100644 index 0000000000000000000000000000000000000000..f064e18c064b308d780abd711cabb5e7baa2e0a4 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/AvroFieldSchemaGenerator.java @@ -0,0 +1,85 @@ +package org.apache.hadoop.hive.metastore.schema.utils; + +import org.apache.avro.Schema; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.serde2.avro.AvroSerdeException; +import org.apache.hadoop.hive.serde2.avro.SchemaToTypeInfo; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; + +import java.util.ArrayList; +import java.util.List; + +public class AvroFieldSchemaGenerator { + final private List columnNames; + final private List columnTypes; + final private List columnComments; + + public AvroFieldSchemaGenerator(Schema schema) throws AvroSerdeException { + verifySchemaIsARecord(schema); + + this.columnNames = generateColumnNames(schema); + this.columnTypes = SchemaToTypeInfo.generateColumnTypes(schema); + this.columnComments = generateColumnComments(schema); + assert columnNames.size() == columnTypes.size(); + } + + private static void verifySchemaIsARecord(Schema schema) throws AvroSerdeException { + if(!schema.getType().equals(Schema.Type.RECORD)) { + throw new AvroSerdeException("Schema for table must be of type RECORD. " + + "Received type: " + schema.getType()); + } + } + + private static List generateColumnNames(Schema schema) { + List fields = schema.getFields(); + List fieldsList = new ArrayList(fields.size()); + + for (Schema.Field field : fields) { + fieldsList.add(field.name()); + } + + return fieldsList; + } + + private static List generateColumnComments(Schema schema) { + List fields = schema.getFields(); + List fieldComments = new ArrayList(fields.size()); + + for (Schema.Field field : fields) { + String fieldComment = field.doc() == null ? "" : field.doc(); + fieldComments.add(fieldComment); + } + + return fieldComments; + } + + public List getFieldSchemas() throws AvroSerdeException { + int len = columnNames.size(); + List fieldSchemas = new ArrayList<>(len); + for(int i = 0; i getFieldsFromAvroSchema(Configuration configuration, + Properties properties) throws AvroSerdeException, IOException { + // Reset member variables so we don't get in a half-constructed state + Schema schema = null; + List columnNames = null; + List columnTypes = null; + + final String columnNameProperty = properties.getProperty(serdeConstants.LIST_COLUMNS); + final String columnTypeProperty = properties.getProperty(serdeConstants.LIST_COLUMN_TYPES); + final String columnCommentProperty = properties.getProperty(LIST_COLUMN_COMMENTS,""); + final String columnNameDelimiter = properties.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? properties + .getProperty(serdeConstants.COLUMN_NAME_DELIMITER) : String.valueOf(SerDeUtils.COMMA); + + if (hasExternalSchema(properties) + || columnNameProperty == null || columnNameProperty.isEmpty() + || columnTypeProperty == null || columnTypeProperty.isEmpty()) { + schema = AvroSchemaUtils.determineSchemaOrThrowException(configuration, properties); + } else { + // Get column names and sort order + columnNames = StringUtils.intern( + Arrays.asList(columnNameProperty.split(columnNameDelimiter))); + columnTypes = StorageSchemaUtils.getTypeInfosFromTypeString(columnTypeProperty); + + schema = getSchemaFromCols(properties, columnNames, columnTypes, columnCommentProperty); + properties.setProperty(AvroTableProperties.SCHEMA_LITERAL.getPropName(), schema.toString()); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Avro schema is " + schema); + } + + if (configuration == null) { + LOG.debug("Configuration null, not inserting schema"); + } else { + configuration.set( + AvroTableProperties.AVRO_SERDE_SCHEMA.getPropName(), schema.toString(false)); + } + return new AvroFieldSchemaGenerator(schema).getFieldSchemas(); + } + + + private static boolean hasExternalSchema(Properties properties) { + return properties.getProperty(AvroTableProperties.SCHEMA_LITERAL.getPropName()) != null + || properties.getProperty(AvroTableProperties.SCHEMA_URL.getPropName()) != null; + } + + private boolean supportedCategories(TypeInfo ti) { + final ObjectInspector.Category c = ti.getCategory(); + return c.equals(ObjectInspector.Category.PRIMITIVE) || + c.equals(ObjectInspector.Category.MAP) || + c.equals(ObjectInspector.Category.LIST) || + c.equals(ObjectInspector.Category.STRUCT) || + c.equals(ObjectInspector.Category.UNION); + } + + /** + * Attempt to determine the schema via the usual means, but do not throw + * an exception if we fail. Instead, signal failure via a special + * schema. + */ + public static Schema determineSchemaOrReturnErrorSchema(Configuration conf, Properties props) { + try { + return AvroSchemaUtils.determineSchemaOrThrowException(conf, props); + } catch(AvroSerdeException he) { + LOG.warn("Encountered AvroSerdeException determining schema. Returning " + + "signal schema to indicate problem", he); + } catch (Exception e) { + LOG.warn("Encountered exception determining schema. Returning signal " + + "schema to indicate problem", e); + } + return SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; + } + + /** + * Determine the schema to that's been provided for Avro serde work. + * @param properties containing a key pointing to the schema, one way or another + * @return schema to use while serdeing the avro file + * @throws IOException if error while trying to read the schema from another location + * @throws AvroSerdeException if unable to find a schema or pointer to it in the properties + */ + public static Schema determineSchemaOrThrowException(Configuration conf, Properties properties) + throws IOException, AvroSerdeException { + String schemaString = properties.getProperty(AvroTableProperties.SCHEMA_LITERAL.getPropName()); + if(schemaString != null && !schemaString.equals(SCHEMA_NONE)) + return AvroSchemaUtils.getSchemaFor(schemaString); + + // Try pulling directly from URL + schemaString = properties.getProperty(AvroTableProperties.SCHEMA_URL.getPropName()); + if (schemaString == null) { + final String columnNameProperty = properties.getProperty(serdeConstants.LIST_COLUMNS); + final String columnTypeProperty = properties.getProperty(serdeConstants.LIST_COLUMN_TYPES); + final String columnCommentProperty = properties.getProperty(LIST_COLUMN_COMMENTS); + if (columnNameProperty == null || columnNameProperty.isEmpty() + || columnTypeProperty == null || columnTypeProperty.isEmpty() ) { + throw new AvroSerdeException(EXCEPTION_MESSAGE); + } + final String columnNameDelimiter = properties.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? properties + .getProperty(serdeConstants.COLUMN_NAME_DELIMITER) : String.valueOf(SerDeUtils.COMMA); + // Get column names and types + List columnNames = Arrays.asList(columnNameProperty.split(columnNameDelimiter)); + List columnTypes = StorageSchemaUtils.getTypeInfosFromTypeString(columnTypeProperty); + + Schema schema = getSchemaFromCols(properties, columnNames, columnTypes, columnCommentProperty); + properties.setProperty(AvroTableProperties.SCHEMA_LITERAL.getPropName(), schema.toString()); + if (conf != null) + conf.set(AvroTableProperties.AVRO_SERDE_SCHEMA.getPropName(), schema.toString(false)); + return schema; + } else if(schemaString.equals(SCHEMA_NONE)) { + throw new AvroSerdeException(EXCEPTION_MESSAGE); + } + + try { + Schema s = getSchemaFromFS(schemaString, conf); + if (s == null) { + //in case schema is not a file system + return AvroSchemaUtils.getSchemaFor(new URL(schemaString)); + } + return s; + } catch (IOException ioe) { + throw new AvroSerdeException("Unable to read schema from given path: " + schemaString, ioe); + } catch (URISyntaxException urie) { + throw new AvroSerdeException("Unable to read schema from given path: " + schemaString, urie); + } + } + + // Protected for testing and so we can pass in a conf for testing. + protected static Schema getSchemaFromFS(String schemaFSUrl, + Configuration conf) throws IOException, URISyntaxException { + FSDataInputStream in = null; + FileSystem fs = null; + try { + fs = FileSystem.get(new URI(schemaFSUrl), conf); + } catch (IOException ioe) { + //return null only if the file system in schema is not recognized + if (LOG.isDebugEnabled()) { + String msg = "Failed to open file system for uri " + schemaFSUrl + " assuming it is not a FileSystem url"; + LOG.debug(msg, ioe); + } + + return null; + } + try { + in = fs.open(new Path(schemaFSUrl)); + Schema s = AvroSchemaUtils.getSchemaFor(in); + return s; + } finally { + if(in != null) in.close(); + } + } + + public static Schema getSchemaFor(File file) { + Schema.Parser parser = new Schema.Parser(); + Schema schema; + try { + schema = parser.parse(file); + } catch (IOException e) { + throw new RuntimeException("Failed to parse Avro schema from " + file.getName(), e); + } + return schema; + } + + public static Schema getSchemaFor(InputStream stream) { + Schema.Parser parser = new Schema.Parser(); + Schema schema; + try { + schema = parser.parse(stream); + } catch (IOException e) { + throw new RuntimeException("Failed to parse Avro schema", e); + } + return schema; + } + + public static Schema getSchemaFor(String str) { + Schema.Parser parser = new Schema.Parser(); + Schema schema = parser.parse(str); + return schema; + } + + public static Schema getSchemaFor(URL url) { + InputStream in = null; + try { + in = url.openStream(); + return getSchemaFor(in); + } catch (Exception e) { + throw new RuntimeException("Failed to parse Avro schema", e); + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException e) { + // Ignore + } + } + } + } + + public static Schema getSchemaFromCols(Properties properties, + List columnNames, List columnTypes, String columnCommentProperty) { + List columnComments; + if (columnCommentProperty == null || columnCommentProperty.isEmpty()) { + columnComments = new ArrayList(); + } else { + //Comments are separated by "\0" in columnCommentProperty, see method getSchema + //in MetaStoreUtils where this string columns.comments is generated + columnComments = Arrays.asList(columnCommentProperty.split("\0")); + + if (LOG.isDebugEnabled()) { + LOG.debug("columnComments is " + columnCommentProperty); + } + } + if (columnNames.size() != columnTypes.size()) { + throw new IllegalArgumentException("getSchemaFromCols initialization failed. Number of column " + + "name and column type differs. columnNames = " + columnNames + ", columnTypes = " + + columnTypes); + } + + final String tableName = properties.getProperty(AvroSerDeConstants.TABLE_NAME); + final String tableComment = properties.getProperty(AvroSerDeConstants.TABLE_COMMENT); + TypeInfoToSchema typeInfoToSchema = new TypeInfoToSchema(); + return typeInfoToSchema.convert(columnNames, columnTypes, columnComments, + properties.getProperty(AvroTableProperties.SCHEMA_NAMESPACE.getPropName()), + properties.getProperty(AvroTableProperties.SCHEMA_NAME.getPropName(), tableName), + properties.getProperty(AvroTableProperties.SCHEMA_DOC.getPropName(), tableComment)); + + } + + /** + * Determine if an Avro schema is of type Union[T, NULL]. Avro supports nullable + * types via a union of type T and null. This is a very common use case. + * As such, we want to silently convert it to just T and allow the value to be null. + * + * When a Hive union type is used with AVRO, the schema type becomes + * Union[NULL, T1, T2, ...]. The NULL in the union should be silently removed + * + * @return true if type represents Union[T, Null], false otherwise + */ + public static boolean isNullableType(Schema schema) { + if (!schema.getType().equals(Schema.Type.UNION)) { + return false; + } + + List itemSchemas = schema.getTypes(); + if (itemSchemas.size() < 2) { + return false; + } + + for (Schema itemSchema : itemSchemas) { + if (Schema.Type.NULL.equals(itemSchema.getType())) { + return true; + } + } + + // [null, null] not allowed, so this check is ok. + return false; + } + + /** + * In a nullable type, get the schema for the non-nullable type. This method + * does no checking that the provides Schema is nullable. + */ + public static Schema getOtherTypeFromNullableType(Schema schema) { + List itemSchemas = new ArrayList<>(); + for (Schema itemSchema : schema.getTypes()) { + if (!Schema.Type.NULL.equals(itemSchema.getType())) { + itemSchemas.add(itemSchema); + } + } + + if (itemSchemas.size() > 1) { + return Schema.createUnion(itemSchemas); + } else { + return itemSchemas.get(0); + } + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/SchemaResolutionProblem.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/SchemaResolutionProblem.java new file mode 100644 index 0000000000000000000000000000000000000000..3f93c01cc375ec40deb88477252ee2e3eb061b81 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/SchemaResolutionProblem.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.schema.utils; + +import org.apache.avro.Schema; + +/** + * This class is copied from SchemaResolutionProblem from the Hive source code as part of metastore + * separation + */ +class SchemaResolutionProblem { + static final String sentinelString = "{\n" + + " \"namespace\": \"org.apache.hadoop.hive\",\n" + + " \"name\": \"CannotDetermineSchemaSentinel\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"ERROR_ERROR_ERROR_ERROR_ERROR_ERROR_ERROR\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"Cannot_determine_schema\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"check\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"schema\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"url\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"and\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"literal\",\n" + + " \"type\":\"string\"\n" + + " }\n" + + " ]\n" + + "}"; + public final static Schema SIGNAL_BAD_SCHEMA = AvroSchemaUtils.getSchemaFor(sentinelString); +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/SerDeUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/SerDeUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..5773ccf0b2387533af1b9f1e33545b03c514591c --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/SerDeUtils.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.schema.utils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.utils.JavaUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.hadoop.hive.serde2.AbstractSerDe; +import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; + +public class SerDeUtils { + + public static final char COMMA = ','; + private static final String FROM_SERIALIZER = "from deserializer"; + private static final Logger LOG = LoggerFactory.getLogger(SerDeUtils.class.getName()); + + private static String determineFieldComment(String comment) { + return (comment == null) ? FROM_SERIALIZER : comment; + } + + /** + * Initializes a SerDe. + * @param deserializer + * @param conf + * @param tblProps + * @param partProps + * @throws SerDeException + */ + public static void initializeSerDeWithoutErrorCheck(Deserializer deserializer, + Configuration conf, Properties tblProps, + Properties partProps) throws SerDeException { + if (deserializer instanceof AbstractSerDe) { + ((AbstractSerDe) deserializer).initialize(conf, tblProps, partProps); + } else { + deserializer.initialize(conf, createOverlayedProperties(tblProps, partProps)); + } + } + + /** + * Returns the union of table and partition properties, + * with partition properties taking precedence. + * @param tblProps + * @param partProps + * @return the overlayed properties + */ + public static Properties createOverlayedProperties(Properties tblProps, Properties partProps) { + Properties props = new Properties(); + props.putAll(tblProps); + if (partProps != null) { + props.putAll(partProps); + } + return props; + } + + /** + * @param tableName name of the table + * @param deserializer deserializer to use + * @return the list of fields + * @throws SerDeException if the serde throws an exception + * @throws MetaException if one of the fields or types in the table is invalid + */ + public static List getFieldsFromDeserializer(String tableName, + Deserializer deserializer) throws SerDeException, MetaException { + ObjectInspector oi = deserializer.getObjectInspector(); + String[] names = tableName.split("\\."); + String last_name = names[names.length - 1]; + for (int i = 1; i < names.length; i++) { + + if (oi instanceof StructObjectInspector) { + StructObjectInspector soi = (StructObjectInspector) oi; + StructField sf = soi.getStructFieldRef(names[i]); + if (sf == null) { + throw new MetaException("Invalid Field " + names[i]); + } else { + oi = sf.getFieldObjectInspector(); + } + } else if (oi instanceof ListObjectInspector + && names[i].equalsIgnoreCase("$elem$")) { + ListObjectInspector loi = (ListObjectInspector) oi; + oi = loi.getListElementObjectInspector(); + } else if (oi instanceof MapObjectInspector + && names[i].equalsIgnoreCase("$key$")) { + MapObjectInspector moi = (MapObjectInspector) oi; + oi = moi.getMapKeyObjectInspector(); + } else if (oi instanceof MapObjectInspector + && names[i].equalsIgnoreCase("$value$")) { + MapObjectInspector moi = (MapObjectInspector) oi; + oi = moi.getMapValueObjectInspector(); + } else { + throw new MetaException("Unknown type for " + names[i]); + } + } + + ArrayList str_fields = new ArrayList<>(); + // rules on how to recurse the ObjectInspector based on its type + if (oi.getCategory() != Category.STRUCT) { + str_fields.add(new FieldSchema(last_name, oi.getTypeName(), + FROM_SERIALIZER)); + } else { + List fields = ((StructObjectInspector) oi) + .getAllStructFieldRefs(); + for (int i = 0; i < fields.size(); i++) { + StructField structField = fields.get(i); + String fieldName = structField.getFieldName(); + String fieldTypeName = structField.getFieldObjectInspector().getTypeName(); + String fieldComment = determineFieldComment(structField.getFieldComment()); + + str_fields.add(new FieldSchema(fieldName, fieldTypeName, fieldComment)); + } + } + return str_fields; + } + + /** + * Initializes a SerDe. + * @param deserializer + * @param conf + * @param tblProps + * @param partProps + * @throws SerDeException + */ + public static void initializeSerDe(Deserializer deserializer, Configuration conf, + Properties tblProps, Properties partProps) + throws SerDeException { + if (deserializer instanceof AbstractSerDe) { + ((AbstractSerDe) deserializer).initialize(conf, tblProps, partProps); + String msg = ((AbstractSerDe) deserializer).getConfigurationErrors(); + if (msg != null && !msg.isEmpty()) { + throw new SerDeException(msg); + } + } else { + deserializer.initialize(conf, createOverlayedProperties(tblProps, partProps)); + } + } + + + /** + * getDeserializer + * + * Get the Deserializer for a table. + * + * @param conf + * - hadoop config + * @param table + * the table + * @return + * Returns instantiated deserializer by looking up class name of deserializer stored in + * storage descriptor of passed in table. Also, initializes the deserializer with schema + * of table. + * @exception MetaException + * if any problems instantiating the Deserializer + * + * todo - this should move somewhere into serde.jar + * + */ + static public Deserializer getDeserializer(Configuration conf, + org.apache.hadoop.hive.metastore.api.Table table, boolean skipConfError) throws + MetaException { + String lib = table.getSd().getSerdeInfo().getSerializationLib(); + if (lib == null) { + return null; + } + return getDeserializer(conf, table, skipConfError, lib); + } + + public static Deserializer getDeserializer(Configuration conf, + org.apache.hadoop.hive.metastore.api.Table table, boolean skipConfError, + String lib) throws MetaException { + try { + Deserializer deserializer = JavaUtils.newInstance(conf.getClassByName(lib). + asSubclass(Deserializer.class), conf); + if (skipConfError) { + SerDeUtils.initializeSerDeWithoutErrorCheck(deserializer, conf, + MetaStoreUtils.getTableMetadata(table), null); + } else { + SerDeUtils.initializeSerDe(deserializer, conf, MetaStoreUtils.getTableMetadata(table), null); + } + return deserializer; + } catch (RuntimeException e) { + throw e; + } catch (Throwable e) { + LOG.error("error in initSerDe: " + e.getClass().getName() + " " + + e.getMessage(), e); + throw new MetaException(e.getClass().getName() + " " + e.getMessage()); + } + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/StorageSchemaUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/StorageSchemaUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..1fc4e1ad5be31e00497f9487324b34c5b013374d --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/StorageSchemaUtils.java @@ -0,0 +1,141 @@ +package org.apache.hadoop.hive.metastore.schema.utils; + +import org.apache.hadoop.hive.common.type.HiveChar; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.HiveCharWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoParser; +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.ByteWritable; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.ShortWritable; +import org.apache.hadoop.io.Text; + +import java.sql.Date; +import java.sql.Timestamp; +import java.util.List; + +public class StorageSchemaUtils { + + public static final PrimitiveTypeEntry binaryTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.BINARY, + serdeConstants.BINARY_TYPE_NAME, byte[].class, byte[].class, BytesWritable.class); + public static final PrimitiveTypeEntry stringTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.STRING, + serdeConstants.STRING_TYPE_NAME, null, String.class, Text.class); + public static final PrimitiveTypeEntry booleanTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN, + serdeConstants.BOOLEAN_TYPE_NAME, Boolean.TYPE, Boolean.class, BooleanWritable.class); + public static final PrimitiveTypeEntry intTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.INT, + serdeConstants.INT_TYPE_NAME, Integer.TYPE, Integer.class, IntWritable.class); + public static final PrimitiveTypeEntry longTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.LONG, + serdeConstants.BIGINT_TYPE_NAME, Long.TYPE, Long.class, LongWritable.class); + public static final PrimitiveTypeEntry floatTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.FLOAT, + serdeConstants.FLOAT_TYPE_NAME, Float.TYPE, Float.class, FloatWritable.class); + public static final PrimitiveTypeEntry voidTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.VOID, + serdeConstants.VOID_TYPE_NAME, Void.TYPE, Void.class, NullWritable.class); + + public static final PrimitiveTypeEntry doubleTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.DOUBLE, + serdeConstants.DOUBLE_TYPE_NAME, Double.TYPE, Double.class, DoubleWritable.class); + public static final PrimitiveTypeEntry byteTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.BYTE, + serdeConstants.TINYINT_TYPE_NAME, Byte.TYPE, Byte.class, ByteWritable.class); + public static final PrimitiveTypeEntry shortTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.SHORT, + serdeConstants.SMALLINT_TYPE_NAME, Short.TYPE, Short.class, ShortWritable.class); + public static final PrimitiveTypeEntry dateTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.DATE, + serdeConstants.DATE_TYPE_NAME, null, Date.class, DateWritable.class); + + public static final PrimitiveTypeEntry timestampTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP, + //serdeConstants.TIMESTAMP_TYPE_NAME, null, Timestamp.class, TimestampWritable.class); + //TODO need to move TimestampWritable to storage-api to make this work + serdeConstants.TIMESTAMP_TYPE_NAME, null, Timestamp.class, null); + public static final PrimitiveTypeEntry timestampTZTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMPLOCALTZ, + //serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME, null, TimestampTZ.class, + //TimestampLocalTZWritable.class); + //TODO need to move TimestampTZ and TimestampLocalTZWritable to storage-api to make this work + serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME, null, null, + null); + public static final PrimitiveTypeEntry intervalYearMonthTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.INTERVAL_YEAR_MONTH, + //serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME, null, HiveIntervalYearMonth.class, + //HiveIntervalYearMonthWritable.class); + //TODO need to move HiveIntervalYearMonth and HiveIntervalYearMonthWritable to storage-api to make this work + serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME, null, null, + null); + public static final PrimitiveTypeEntry intervalDayTimeTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.INTERVAL_DAY_TIME, + serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME, null, HiveIntervalDayTime.class, + //HiveIntervalDayTimeWritable.class); + //TODO need to move HiveIntervalYearMonth and HiveIntervalDayTimeWritable to storage-api to make this work + null); + public static final PrimitiveTypeEntry decimalTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.DECIMAL, + serdeConstants.DECIMAL_TYPE_NAME, null, HiveDecimal.class, HiveDecimalWritable.class); + public static final PrimitiveTypeEntry varcharTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.VARCHAR, + serdeConstants.VARCHAR_TYPE_NAME, null, HiveVarchar.class, HiveVarcharWritable.class); + public static final PrimitiveTypeEntry charTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.CHAR, + serdeConstants.CHAR_TYPE_NAME, null, HiveChar.class, HiveCharWritable.class); + + // The following is a complex type for special handling + public static final PrimitiveTypeEntry unknownTypeEntry = + new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.UNKNOWN, "unknown", null, + Object.class, null); + + + static { + PrimitiveTypeEntry.registerType(binaryTypeEntry); + PrimitiveTypeEntry.registerType(stringTypeEntry); + PrimitiveTypeEntry.registerType(charTypeEntry); + PrimitiveTypeEntry.registerType(varcharTypeEntry); + PrimitiveTypeEntry.registerType(booleanTypeEntry); + PrimitiveTypeEntry.registerType(intTypeEntry); + PrimitiveTypeEntry.registerType(longTypeEntry); + PrimitiveTypeEntry.registerType(floatTypeEntry); + PrimitiveTypeEntry.registerType(voidTypeEntry); + PrimitiveTypeEntry.registerType(doubleTypeEntry); + PrimitiveTypeEntry.registerType(byteTypeEntry); + PrimitiveTypeEntry.registerType(shortTypeEntry); + PrimitiveTypeEntry.registerType(dateTypeEntry); + PrimitiveTypeEntry.registerType(timestampTypeEntry); + PrimitiveTypeEntry.registerType(timestampTZTypeEntry); + PrimitiveTypeEntry.registerType(intervalYearMonthTypeEntry); + PrimitiveTypeEntry.registerType(intervalDayTimeTypeEntry); + PrimitiveTypeEntry.registerType(decimalTypeEntry); + PrimitiveTypeEntry.registerType(unknownTypeEntry); + } + + + public static final char COMMA = ','; + public static List getTypeInfosFromTypeString(String columnTypeProperty) { + return new TypeInfoParser(columnTypeProperty).parseTypeInfos(); + } + + private static final String FROM_STORAGE_SCHEMA_READER = "generated by storage schema reader"; + public static String determineFieldComment(String comment) { + return (comment == null) ? FROM_STORAGE_SCHEMA_READER : comment; + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java index b08d9fd71f6c27abcda5561adeef755d6f3fc003..60682e24afc85b73a49bb83b8650dea43b669016 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java @@ -17,16 +17,38 @@ */ package org.apache.hadoop.hive.metastore.utils; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.MetaException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.reflect.Constructor; +import java.lang.reflect.Method; import java.net.InetAddress; import java.net.UnknownHostException; public class JavaUtils { public static final Logger LOG = LoggerFactory.getLogger(JavaUtils.class); + private static final Method configureMethod; + private static final Class jobConfClass, jobConfigurableClass; + + static { + Class jobConfClassLocal, jobConfigurableClassLocal; + Method configureMethodLocal; + try { + jobConfClassLocal = Class.forName("org.apache.hadoop.mapred.JobConf"); + jobConfigurableClassLocal = Class.forName("org.apache.hadoop.mapred.JobConfigurable"); + configureMethodLocal = jobConfigurableClassLocal.getMethod("configure", jobConfClassLocal); + } catch (Throwable t) { + // Meh. + jobConfClassLocal = jobConfigurableClassLocal = null; + configureMethodLocal = null; + } + jobConfClass = jobConfClassLocal; + jobConfigurableClass = jobConfigurableClassLocal; + configureMethod = configureMethodLocal; + } /** * Standard way of getting classloader in Hive code (outside of Hadoop). @@ -100,8 +122,55 @@ public static ClassLoader getClassLoader() { throw new RuntimeException("Unable to instantiate " + theClass.getName(), e); } } + private static final Class[] EMPTY_ARRAY = new Class[] {}; + /** + * Create an object for the given class and initialize it from conf + * @param theClass class of which an object is created + * @param conf Configuration + * @return a new object + */ + @SuppressWarnings("unchecked") + public static T newInstance(Class theClass, Configuration conf) { + T result; + try { + // TODO Do we need a constructor cache like Hive here? + Constructor ctor = theClass.getDeclaredConstructor(EMPTY_ARRAY); + ctor.setAccessible(true); + result = (T)ctor.newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } + setConf(result, conf); + return result; + } /** + * Check and set 'configuration' if necessary. + * + * @param theObject object for which to set configuration + * @param conf Configuration + */ + public static void setConf(Object theObject, Configuration conf) { + if (conf != null) { + if (theObject instanceof Configurable) { + ((Configurable) theObject).setConf(conf); + } + setJobConf(theObject, conf); + } + } + + private static void setJobConf(Object theObject, Configuration conf) { + if (configureMethod == null) return; + try { + if (jobConfClass.isAssignableFrom(conf.getClass()) && + jobConfigurableClass.isAssignableFrom(theObject.getClass())) { + configureMethod.invoke(theObject, conf); + } + } catch (Exception e) { + throw new RuntimeException("Error in configuring object", e); + } + } + /** * @return name of current host */ public static String hostname() { diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index cde34bcf42bdb49f7e4d11b9274337ad1b4c3285..8e52779d8fc6f53dc0fe64467c4f40b1d1dc3eb9 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; +import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.security.SaslRpcServer; import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; import org.apache.hadoop.security.authorize.ProxyUsers; @@ -1560,4 +1561,5 @@ public static boolean isMaterializedViewTable(Table table) { } return cols; } + } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerDeConstants.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerDeConstants.java new file mode 100644 index 0000000000000000000000000000000000000000..e875c13a1af3733fcb76f941144519a77c46c8c6 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerDeConstants.java @@ -0,0 +1,25 @@ +package org.apache.hadoop.hive.serde2.avro; + +/** + * This class contains some of the constants which are specific to AvroSerDe + * They should always match with the constants defined in AvroSerDe.java in Hive Source code. These + * constants were copied as part of separating metastore from Hive. + */ +public class AvroSerDeConstants { + public static final String TABLE_NAME = "name"; + public static final String TABLE_COMMENT = "comment"; + public static final String LIST_COLUMN_COMMENTS = "columns.comments"; + + public static final String DECIMAL_TYPE_NAME = "decimal"; + public static final String CHAR_TYPE_NAME = "char"; + public static final String VARCHAR_TYPE_NAME = "varchar"; + public static final String DATE_TYPE_NAME = "date"; + public static final String TIMESTAMP_TYPE_NAME = "timestamp-millis"; + public static final String AVRO_PROP_LOGICAL_TYPE = "logicalType"; + public static final String AVRO_PROP_PRECISION = "precision"; + public static final String AVRO_PROP_SCALE = "scale"; + public static final String AVRO_PROP_MAX_LENGTH = "maxLength"; + public static final String AVRO_STRING_TYPE_NAME = "string"; + public static final String AVRO_INT_TYPE_NAME = "int"; + public static final String AVRO_LONG_TYPE_NAME = "long"; +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeException.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeException.java new file mode 100644 index 0000000000000000000000000000000000000000..bbf4cc1078f2a9d1395723c395228db865ca850d --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeException.java @@ -0,0 +1,11 @@ +package org.apache.hadoop.hive.serde2.avro; + +public class AvroSerdeException extends Exception { + public AvroSerdeException(String s, Exception ex) { + super(s, ex); + } + + public AvroSerdeException(String msg) { + super(msg); + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java new file mode 100644 index 0000000000000000000000000000000000000000..acc693c7b7cfecff08ce70fc918ed2276c0518f0 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.avro; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Cache for objects whose creation only depends on some other set of objects and therefore can be + * used against other equivalent versions of those objects. Essentially memoizes instance creation. + * + * @param Object that determines the instance. The cache uses this object as a key for + * its hash which is why it is imperative to have appropriate equals and hashcode + * implementation for this object for the cache to work properly + * @param Instance that will be created from SeedObject. + */ +public abstract class InstanceCache { + private static final Logger LOG = LoggerFactory.getLogger(InstanceCache.class); + Map cache = new HashMap(); + + public InstanceCache() {} + + /** + * Retrieve (or create if it doesn't exist) the correct Instance for this + * SeedObject + */ + public Instance retrieve(SeedObject hv) throws AvroSerdeException { + return retrieve(hv, null); + } + + /** + * Retrieve (or create if it doesn't exist) the correct Instance for this + * SeedObject using 'seenSchemas' to resolve circular references + */ + public synchronized Instance retrieve(SeedObject hv, + Set seenSchemas) throws AvroSerdeException { + if(LOG.isDebugEnabled()) LOG.debug("Checking for hv: " + hv.toString()); + + if(cache.containsKey(hv)) { + if(LOG.isDebugEnabled()) LOG.debug("Returning cache result."); + return cache.get(hv); + } + + if(LOG.isDebugEnabled()) LOG.debug("Creating new instance and storing in cache"); + + Instance instance = makeInstance(hv, seenSchemas); + cache.put(hv, instance); + return instance; + } + + protected abstract Instance makeInstance(SeedObject hv, + Set seenSchemas) throws AvroSerdeException; +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..29298b0e2c2964634b029c590357e6c91d102635 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.avro; + +import static org.apache.avro.Schema.Type.BOOLEAN; +import static org.apache.avro.Schema.Type.BYTES; +import static org.apache.avro.Schema.Type.DOUBLE; +import static org.apache.avro.Schema.Type.FIXED; +import static org.apache.avro.Schema.Type.FLOAT; +import static org.apache.avro.Schema.Type.INT; +import static org.apache.avro.Schema.Type.LONG; +import static org.apache.avro.Schema.Type.NULL; +import static org.apache.avro.Schema.Type.STRING; + +import org.apache.avro.Schema; +import org.apache.hadoop.hive.metastore.schema.utils.AvroSchemaUtils; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfoValidationUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Hashtable; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Convert an Avro Schema to a Hive TypeInfo. This class is copied from Hive source code for + * standalone metastore + */ +public class SchemaToTypeInfo { + // Conversion of Avro primitive types to Hive primitive types + // Avro Hive + // Null + // boolean boolean check + // int int check + // long bigint check + // float double check + // double double check + // bytes binary check + // fixed binary check + // string string check + // tinyint + // smallint + + // Map of Avro's primitive types to Hives (for those that are supported by both) + private static final Map primitiveTypeToTypeInfo = initTypeMap(); + private static Map initTypeMap() { + Map theMap = new Hashtable(); + theMap.put(NULL, TypeInfoFactory.getPrimitiveTypeInfo("void")); + theMap.put(BOOLEAN, TypeInfoFactory.getPrimitiveTypeInfo("boolean")); + theMap.put(INT, TypeInfoFactory.getPrimitiveTypeInfo("int")); + theMap.put(LONG, TypeInfoFactory.getPrimitiveTypeInfo("bigint")); + theMap.put(FLOAT, TypeInfoFactory.getPrimitiveTypeInfo("float")); + theMap.put(DOUBLE, TypeInfoFactory.getPrimitiveTypeInfo("double")); + theMap.put(BYTES, TypeInfoFactory.getPrimitiveTypeInfo("binary")); + theMap.put(FIXED, TypeInfoFactory.getPrimitiveTypeInfo("binary")); + theMap.put(STRING, TypeInfoFactory.getPrimitiveTypeInfo("string")); + return Collections.unmodifiableMap(theMap); + } + + /** + * Generate a list of of TypeInfos from an Avro schema. This method is + * currently public due to some weirdness in deserializing unions, but + * will be made private once that is resolved. + * @param schema Schema to generate field types for + * @return List of TypeInfos, each element of which is a TypeInfo derived + * from the schema. + * @throws AvroSerdeException for problems during conversion. + */ + public static List generateColumnTypes(Schema schema) throws AvroSerdeException { + return generateColumnTypes (schema, null); + } + + /** + * Generate a list of of TypeInfos from an Avro schema. This method is + * currently public due to some weirdness in deserializing unions, but + * will be made private once that is resolved. + * @param schema Schema to generate field types for + * @param seenSchemas stores schemas processed in the parsing done so far, + * helping to resolve circular references in the schema + * @return List of TypeInfos, each element of which is a TypeInfo derived + * from the schema. + * @throws AvroSerdeException for problems during conversion. + */ + public static List generateColumnTypes(Schema schema, + Set seenSchemas) throws AvroSerdeException { + List fields = schema.getFields(); + + List types = new ArrayList(fields.size()); + + for (Schema.Field field : fields) { + types.add(generateTypeInfo(field.schema(), seenSchemas)); + } + + return types; + } + + static InstanceCache typeInfoCache = new InstanceCache() { + @Override + protected TypeInfo makeInstance(Schema s, + Set seenSchemas) + throws AvroSerdeException { + return generateTypeInfoWorker(s, seenSchemas); + } + }; + /** + * Convert an Avro Schema into an equivalent Hive TypeInfo. + * @param schema to record. Must be of record type. + * @param seenSchemas stores schemas processed in the parsing done so far, + * helping to resolve circular references in the schema + * @return TypeInfo matching the Avro schema + * @throws AvroSerdeException for any problems during conversion. + */ + public static TypeInfo generateTypeInfo(Schema schema, + Set seenSchemas) throws AvroSerdeException { + // For bytes type, it can be mapped to decimal. + Schema.Type type = schema.getType(); + if (type == BYTES && AvroSerDeConstants.DECIMAL_TYPE_NAME + .equalsIgnoreCase(schema.getProp(AvroSerDeConstants.AVRO_PROP_LOGICAL_TYPE))) { + int precision = 0; + int scale = 0; + try { + precision = schema.getJsonProp(AvroSerDeConstants.AVRO_PROP_PRECISION).getIntValue(); + scale = schema.getJsonProp(AvroSerDeConstants.AVRO_PROP_SCALE).getIntValue(); + } catch (Exception ex) { + throw new AvroSerdeException("Failed to obtain scale value from file schema: " + schema, ex); + } + + try { + PrimitiveTypeInfoValidationUtils.validateParameter(precision, scale); + } catch (Exception ex) { + throw new AvroSerdeException("Invalid precision or scale for decimal type", ex); + } + + return TypeInfoFactory.getDecimalTypeInfo(precision, scale); + } + + if (type == STRING && + AvroSerDeConstants.CHAR_TYPE_NAME.equalsIgnoreCase(schema.getProp(AvroSerDeConstants.AVRO_PROP_LOGICAL_TYPE))) { + int maxLength = 0; + try { + maxLength = schema.getJsonProp(AvroSerDeConstants.AVRO_PROP_MAX_LENGTH).getValueAsInt(); + } catch (Exception ex) { + throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex); + } + return TypeInfoFactory.getCharTypeInfo(maxLength); + } + + if (type == STRING && AvroSerDeConstants.VARCHAR_TYPE_NAME + .equalsIgnoreCase(schema.getProp(AvroSerDeConstants.AVRO_PROP_LOGICAL_TYPE))) { + int maxLength = 0; + try { + maxLength = schema.getJsonProp(AvroSerDeConstants.AVRO_PROP_MAX_LENGTH).getValueAsInt(); + } catch (Exception ex) { + throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex); + } + return TypeInfoFactory.getVarcharTypeInfo(maxLength); + } + + if (type == INT && + AvroSerDeConstants.DATE_TYPE_NAME.equals(schema.getProp(AvroSerDeConstants.AVRO_PROP_LOGICAL_TYPE))) { + return TypeInfoFactory.dateTypeInfo; + } + + if (type == LONG && + AvroSerDeConstants.TIMESTAMP_TYPE_NAME.equals(schema.getProp(AvroSerDeConstants.AVRO_PROP_LOGICAL_TYPE))) { + return TypeInfoFactory.timestampTypeInfo; + } + + return typeInfoCache.retrieve(schema, seenSchemas); + } + + private static TypeInfo generateTypeInfoWorker(Schema schema, + Set seenSchemas) throws AvroSerdeException { + // Avro requires NULLable types to be defined as unions of some type T + // and NULL. This is annoying and we're going to hide it from the user. + if(AvroSchemaUtils.isNullableType(schema)) { + return generateTypeInfo( + AvroSchemaUtils.getOtherTypeFromNullableType(schema), seenSchemas); + } + + Schema.Type type = schema.getType(); + if(primitiveTypeToTypeInfo.containsKey(type)) { + return primitiveTypeToTypeInfo.get(type); + } + + switch(type) { + case RECORD: return generateRecordTypeInfo(schema, seenSchemas); + case MAP: return generateMapTypeInfo(schema, seenSchemas); + case ARRAY: return generateArrayTypeInfo(schema, seenSchemas); + case UNION: return generateUnionTypeInfo(schema, seenSchemas); + case ENUM: return generateEnumTypeInfo(schema); + default: throw new AvroSerdeException("Do not yet support: " + schema); + } + } + + private static TypeInfo generateRecordTypeInfo(Schema schema, + Set seenSchemas) throws AvroSerdeException { + assert schema.getType().equals(Schema.Type.RECORD); + + if (seenSchemas == null) { + seenSchemas = Collections.newSetFromMap(new IdentityHashMap()); + } else if (seenSchemas.contains(schema)) { + throw new AvroSerdeException( + "Recursive schemas are not supported. Recursive schema was " + schema + .getFullName()); + } + seenSchemas.add(schema); + + List fields = schema.getFields(); + List fieldNames = new ArrayList(fields.size()); + List typeInfos = new ArrayList(fields.size()); + + for(int i = 0; i < fields.size(); i++) { + fieldNames.add(i, fields.get(i).name()); + typeInfos.add(i, generateTypeInfo(fields.get(i).schema(), seenSchemas)); + } + + return TypeInfoFactory.getStructTypeInfo(fieldNames, typeInfos); + } + + /** + * Generate a TypeInfo for an Avro Map. This is made slightly simpler in that + * Avro only allows maps with strings for keys. + */ + private static TypeInfo generateMapTypeInfo(Schema schema, + Set seenSchemas) throws AvroSerdeException { + assert schema.getType().equals(Schema.Type.MAP); + Schema valueType = schema.getValueType(); + TypeInfo ti = generateTypeInfo(valueType, seenSchemas); + + return TypeInfoFactory.getMapTypeInfo(TypeInfoFactory.getPrimitiveTypeInfo("string"), ti); + } + + private static TypeInfo generateArrayTypeInfo(Schema schema, + Set seenSchemas) throws AvroSerdeException { + assert schema.getType().equals(Schema.Type.ARRAY); + Schema itemsType = schema.getElementType(); + TypeInfo itemsTypeInfo = generateTypeInfo(itemsType, seenSchemas); + + return TypeInfoFactory.getListTypeInfo(itemsTypeInfo); + } + + private static TypeInfo generateUnionTypeInfo(Schema schema, + Set seenSchemas) throws AvroSerdeException { + assert schema.getType().equals(Schema.Type.UNION); + List types = schema.getTypes(); + + + List typeInfos = new ArrayList(types.size()); + + for(Schema type : types) { + typeInfos.add(generateTypeInfo(type, seenSchemas)); + } + + return TypeInfoFactory.getUnionTypeInfo(typeInfos); + } + + // Hive doesn't have an Enum type, so we're going to treat them as Strings. + // During the deserialize/serialize stage we'll check for enumness and + // convert as such. + private static TypeInfo generateEnumTypeInfo(Schema schema) { + assert schema.getType().equals(Schema.Type.ENUM); + + return TypeInfoFactory.getPrimitiveTypeInfo("string"); + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java new file mode 100644 index 0000000000000000000000000000000000000000..a3a00b3c852c8d5d087905350a09a7b9e92d29c1 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.avro; + +import org.apache.avro.Schema; +import org.apache.hadoop.hive.metastore.schema.utils.AvroSchemaUtils; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.*; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.node.JsonNodeFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * This class contains util methods to convert TypeInfo to Avro schema and vice-versa. This class + * is copied from TypeInfoToSchema from Hive source code. + */ +public class TypeInfoToSchema { + + private long recordCounter = 0; + + /** + * Converts Hive schema to avro schema + * + * @param columnNames Names of the hive columns + * @param columnTypes Hive Column types + * @param namespace Namespace of Avro schema + * @param name Avro schema name + * @param doc Avro schema doc + * @return Avro Schema + */ + public Schema convert(List columnNames, List columnTypes, + List columnComments, String namespace, String name, String doc) { + + List fields = new ArrayList(); + for (int i = 0; i < columnNames.size(); ++i) { + final String comment = columnComments.size() > i ? columnComments.get(i) : null; + final Schema.Field avroField = createAvroField(columnNames.get(i), columnTypes.get(i), + comment); + fields.addAll(getFields(avroField)); + } + + if (name == null || name.isEmpty()) { + name = "baseRecord"; + } + + Schema avroSchema = Schema.createRecord(name, doc, namespace, false); + avroSchema.setFields(fields); + return avroSchema; + } + + private Schema.Field createAvroField(String name, TypeInfo typeInfo, String comment) { + return new Schema.Field(name, createAvroSchema(typeInfo), comment, null); + } + + private Schema createAvroSchema(TypeInfo typeInfo) { + Schema schema = null; + switch (typeInfo.getCategory()) { + case PRIMITIVE: + schema = createAvroPrimitive(typeInfo); + break; + case LIST: + schema = createAvroArray(typeInfo); + break; + case MAP: + schema = createAvroMap(typeInfo); + break; + case STRUCT: + schema = createAvroRecord(typeInfo); + break; + case UNION: + schema = createAvroUnion(typeInfo); + break; + } + + return wrapInUnionWithNull(schema); + } + + private Schema createAvroPrimitive(TypeInfo typeInfo) { + PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo; + Schema schema; + switch (primitiveTypeInfo.getPrimitiveCategory()) { + case STRING: + schema = Schema.create(Schema.Type.STRING); + break; + case CHAR: + schema = AvroSchemaUtils.getSchemaFor("{" + + "\"type\":\"" + AvroSerDeConstants.AVRO_STRING_TYPE_NAME + "\"," + + "\"logicalType\":\"" + AvroSerDeConstants.CHAR_TYPE_NAME + "\"," + + "\"maxLength\":" + ((CharTypeInfo) typeInfo).getLength() + "}"); + break; + case VARCHAR: + schema = AvroSchemaUtils.getSchemaFor("{" + + "\"type\":\"" + AvroSerDeConstants.AVRO_STRING_TYPE_NAME + "\"," + + "\"logicalType\":\"" + AvroSerDeConstants.VARCHAR_TYPE_NAME + "\"," + + "\"maxLength\":" + ((VarcharTypeInfo) typeInfo).getLength() + "}"); + break; + case BINARY: + schema = Schema.create(Schema.Type.BYTES); + break; + case BYTE: + schema = Schema.create(Schema.Type.INT); + break; + case SHORT: + schema = Schema.create(Schema.Type.INT); + break; + case INT: + schema = Schema.create(Schema.Type.INT); + break; + case LONG: + schema = Schema.create(Schema.Type.LONG); + break; + case FLOAT: + schema = Schema.create(Schema.Type.FLOAT); + break; + case DOUBLE: + schema = Schema.create(Schema.Type.DOUBLE); + break; + case BOOLEAN: + schema = Schema.create(Schema.Type.BOOLEAN); + break; + case DECIMAL: + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo; + String precision = String.valueOf(decimalTypeInfo.precision()); + String scale = String.valueOf(decimalTypeInfo.scale()); + schema = AvroSchemaUtils.getSchemaFor("{" + + "\"type\":\"bytes\"," + + "\"logicalType\":\"decimal\"," + + "\"precision\":" + precision + "," + + "\"scale\":" + scale + "}"); + break; + case DATE: + schema = AvroSchemaUtils.getSchemaFor("{" + + "\"type\":\"" + AvroSerDeConstants.AVRO_INT_TYPE_NAME + "\"," + + "\"logicalType\":\"" + AvroSerDeConstants.DATE_TYPE_NAME + "\"}"); + break; + case TIMESTAMP: + schema = AvroSchemaUtils.getSchemaFor("{" + + "\"type\":\"" + AvroSerDeConstants.AVRO_LONG_TYPE_NAME + "\"," + + "\"logicalType\":\"" + AvroSerDeConstants.TIMESTAMP_TYPE_NAME + "\"}"); + break; + case VOID: + schema = Schema.create(Schema.Type.NULL); + break; + default: + throw new UnsupportedOperationException(typeInfo + " is not supported."); + } + return schema; + } + + private Schema createAvroUnion(TypeInfo typeInfo) { + List childSchemas = new ArrayList(); + for (TypeInfo childTypeInfo : ((UnionTypeInfo) typeInfo).getAllUnionObjectTypeInfos()) { + final Schema childSchema = createAvroSchema(childTypeInfo); + if (childSchema.getType() == Schema.Type.UNION) { + childSchemas.addAll(childSchema.getTypes()); + } else { + childSchemas.add(childSchema); + } + } + + return Schema.createUnion(removeDuplicateNullSchemas(childSchemas)); + } + + private Schema createAvroRecord(TypeInfo typeInfo) { + List childFields = new ArrayList(); + + final List allStructFieldNames = + ((StructTypeInfo) typeInfo).getAllStructFieldNames(); + final List allStructFieldTypeInfos = + ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos(); + if (allStructFieldNames.size() != allStructFieldTypeInfos.size()) { + throw new IllegalArgumentException("Failed to generate avro schema from hive schema. " + + "name and column type differs. names = " + allStructFieldNames + ", types = " + + allStructFieldTypeInfos); + } + + for (int i = 0; i < allStructFieldNames.size(); ++i) { + final TypeInfo childTypeInfo = allStructFieldTypeInfos.get(i); + final Schema.Field grandChildSchemaField = createAvroField(allStructFieldNames.get(i), + childTypeInfo, childTypeInfo.toString()); + final List grandChildFields = getFields(grandChildSchemaField); + childFields.addAll(grandChildFields); + } + + Schema recordSchema = Schema.createRecord("record_" + recordCounter, typeInfo.toString(), + null, false); + ++recordCounter; + recordSchema.setFields(childFields); + return recordSchema; + } + + private Schema createAvroMap(TypeInfo typeInfo) { + TypeInfo keyTypeInfo = ((MapTypeInfo) typeInfo).getMapKeyTypeInfo(); + if (((PrimitiveTypeInfo) keyTypeInfo).getPrimitiveCategory() + != PrimitiveObjectInspector.PrimitiveCategory.STRING) { + throw new UnsupportedOperationException("Key of Map can only be a String"); + } + + TypeInfo valueTypeInfo = ((MapTypeInfo) typeInfo).getMapValueTypeInfo(); + Schema valueSchema = createAvroSchema(valueTypeInfo); + + return Schema.createMap(valueSchema); + } + + private Schema createAvroArray(TypeInfo typeInfo) { + ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; + Schema listSchema = createAvroSchema(listTypeInfo.getListElementTypeInfo()); + return Schema.createArray(listSchema); + } + + private List getFields(Schema.Field schemaField) { + List fields = new ArrayList(); + + JsonNode nullDefault = JsonNodeFactory.instance.nullNode(); + if (schemaField.schema().getType() == Schema.Type.RECORD) { + for (Schema.Field field : schemaField.schema().getFields()) { + fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), nullDefault)); + } + } else { + fields.add(new Schema.Field(schemaField.name(), schemaField.schema(), schemaField.doc(), + nullDefault)); + } + + return fields; + } + + private Schema wrapInUnionWithNull(Schema schema) { + Schema wrappedSchema = schema; + switch (schema.getType()) { + case NULL: + break; + case UNION: + List existingSchemas = removeDuplicateNullSchemas(schema.getTypes()); + wrappedSchema = Schema.createUnion(existingSchemas); + break; + default: + wrappedSchema = Schema.createUnion(Arrays.asList(Schema.create(Schema.Type.NULL), schema)); + } + + return wrappedSchema; + } + + private List removeDuplicateNullSchemas(List childSchemas) { + List prunedSchemas = new ArrayList(); + boolean isNullPresent = false; + for (Schema schema : childSchemas) { + if (schema.getType() == Schema.Type.NULL) { + isNullPresent = true; + } else { + prunedSchemas.add(schema); + } + } + if (isNullPresent) { + prunedSchemas.add(0, Schema.create(Schema.Type.NULL)); + } + + return prunedSchemas; + } +} \ No newline at end of file diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/schema/reader/TestDefaultStorageSchemaReader.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/schema/reader/TestDefaultStorageSchemaReader.java new file mode 100644 index 0000000000000000000000000000000000000000..8feb4f2f1a24e94873b8502c9fc410d0b422b415 --- /dev/null +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/schema/reader/TestDefaultStorageSchemaReader.java @@ -0,0 +1,579 @@ +package org.apache.hadoop.hive.metastore.schema.reader; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.ColumnType; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; +import org.apache.hadoop.hive.metastore.TestHiveMetaStore; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.Type; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.schema.utils.AvroSchemaUtils; +import org.apache.hadoop.util.StringUtils; +import org.apache.thrift.TException; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +public class TestDefaultStorageSchemaReader { + private static final Logger LOG = LoggerFactory.getLogger(TestDefaultStorageSchemaReader.class); + private static final String TEST_DB_NAME = "TEST_DB"; + private static final String TEST_TABLE_NAME = "TEST_TABLE"; + private HiveMetaStoreClient client; + private Configuration conf; + private Warehouse warehouse; + private static final int DEFAULT_LIMIT_PARTITION_REQUEST = 100; + private static final String AVRO_SERIALIZATION_LIB = + "org.apache.hadoop.hive.serde2.avro.AvroSerDe"; + + // These schemata are used in other tests + static public final String MAP_WITH_PRIMITIVE_VALUE_TYPE = "{\n" + + " \"namespace\": \"testing\",\n" + + " \"name\": \"oneMap\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aMap\",\n" + + " \"type\":{\"type\":\"map\",\n" + + " \"values\":\"long\"}\n" + + "\t}\n" + + " ]\n" + + "}"; + static public final String ARRAY_WITH_PRIMITIVE_ELEMENT_TYPE = "{\n" + + " \"namespace\": \"testing\",\n" + + " \"name\": \"oneArray\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"anArray\",\n" + + " \"type\":{\"type\":\"array\",\n" + + " \"items\":\"string\"}\n" + + "\t}\n" + + " ]\n" + + "}"; + public static final String RECORD_SCHEMA = "{\n" + + " \"namespace\": \"testing.test.mctesty\",\n" + + " \"name\": \"oneRecord\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aRecord\",\n" + + " \"type\":{\"type\":\"record\",\n" + + " \"name\":\"recordWithinARecord\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"int1\",\n" + + " \"type\":\"int\"\n" + + " },\n" + + " {\n" + + " \"name\":\"boolean1\",\n" + + " \"type\":\"boolean\"\n" + + " },\n" + + " {\n" + + " \"name\":\"long1\",\n" + + " \"type\":\"long\"\n" + + " }\n" + + " ]}\n" + + " }\n" + + " ]\n" + + "}"; + public static final String NULLABLE_RECORD_SCHEMA = "[\"null\", " + RECORD_SCHEMA + "]"; + public static final String UNION_SCHEMA = "{\n" + + " \"namespace\": \"test.a.rossa\",\n" + + " \"name\": \"oneUnion\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aUnion\",\n" + + " \"type\":[\"int\", \"string\"]\n" + + " }\n" + + " ]\n" + + "}"; + public static final String UNION_SCHEMA_2 = "{\n" + + " \"namespace\": \"test.a.rossa\",\n" + + " \"name\": \"oneUnion\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aUnion\",\n" + + " \"type\":[\"null\", \"int\", \"string\"]\n" + + " }\n" + + " ]\n" + + "}"; + public static final String UNION_SCHEMA_3 = "{\n" + + " \"namespace\": \"test.a.rossa\",\n" + + " \"name\": \"oneUnion\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aUnion\",\n" + + " \"type\":[\"float\",\"int\"]\n" + + " }\n" + + " ]\n" + + "}"; + public static final String UNION_SCHEMA_4 = "{\n" + + " \"namespace\": \"test.a.rossa\",\n" + + " \"name\": \"oneUnion\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aUnion\",\n" + + " \"type\":[\"int\",\"float\",\"long\"]\n" + + " }\n" + + " ]\n" + + "}"; + public static final String ENUM_SCHEMA = "{\n" + + " \"namespace\": \"clever.namespace.name.in.space\",\n" + + " \"name\": \"oneEnum\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"baddies\",\n" + + " \"type\":{\"type\":\"enum\",\"name\":\"villians\", \"symbols\": " + + "[\"DALEKS\", \"CYBERMEN\", \"SLITHEEN\", \"JAGRAFESS\"]}\n" + + " \n" + + " \n" + + " }\n" + + " ]\n" + + "}"; + public static final String FIXED_SCHEMA = "{\n" + + " \"namespace\": \"ecapseman\",\n" + + " \"name\": \"oneFixed\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"hash\",\n" + + " \"type\":{\"type\": \"fixed\", \"name\": \"MD5\", \"size\": 16}\n" + + " }\n" + + " ]\n" + + "}"; + public static final String NULLABLE_STRING_SCHEMA = "{\n" + + " \"type\": \"record\", \n" + + " \"name\": \"nullableUnionTest\",\n" + + " \"fields\" : [\n" + + " {\"name\":\"nullableString\", \"type\":[\"null\", \"string\"]}\n" + + " ]\n" + + "}"; + public static final String MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA = "{\n" + + " \"namespace\": \"testing\",\n" + + " \"name\": \"mapWithNullableUnionTest\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aMap\",\n" + + " \"type\":{\"type\":\"map\",\n" + + " \"values\":[\"null\",\"long\"]}\n" + + "\t}\n" + + " ]\n" + + "}"; + public static final String NULLABLE_ENUM_SCHEMA = "{\n" + + " \"namespace\": \"clever.namespace.name.in.space\",\n" + + " \"name\": \"nullableUnionTest\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"nullableEnum\",\n" + + " \"type\": [\"null\", {\"type\":\"enum\",\"name\":\"villians\", \"symbols\": " + + "[\"DALEKS\", \"CYBERMEN\", \"SLITHEEN\", \"JAGRAFESS\"]}]\n" + + " \n" + + " \n" + + " }\n" + + " ]\n" + + "}"; + public static final String BYTES_SCHEMA = "{\n" + + " \"type\": \"record\", \n" + + " \"name\": \"bytesTest\",\n" + + " \"fields\" : [\n" + + " {\"name\":\"bytesField\", \"type\":\"bytes\"}\n" + + " ]\n" + + "}"; + + public static final String KITCHEN_SINK_SCHEMA = "{\n" + + " \"namespace\": \"org.apache.hadoop.hive\",\n" + + " \"name\": \"kitchsink\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"string1\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"string2\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"int1\",\n" + + " \"type\":\"int\"\n" + + " },\n" + + " {\n" + + " \"name\":\"boolean1\",\n" + + " \"type\":\"boolean\"\n" + + " },\n" + + " {\n" + + " \"name\":\"long1\",\n" + + " \"type\":\"long\"\n" + + " },\n" + + " {\n" + + " \"name\":\"float1\",\n" + + " \"type\":\"float\"\n" + + " },\n" + + " {\n" + + " \"name\":\"double1\",\n" + + " \"type\":\"double\"\n" + + " },\n" + + " {\n" + + " \"name\":\"inner_record1\",\n" + + " \"type\":{ \"type\":\"record\",\n" + + " \"name\":\"inner_record1_impl\",\n" + + " \"fields\": [\n" + + " {\"name\":\"int_in_inner_record1\",\n" + + " \"type\":\"int\"},\n" + + " {\"name\":\"string_in_inner_record1\",\n" + + " \"type\":\"string\"}\n" + + " ]\n" + + " }\n" + + " },\n" + + " {\n" + + " \"name\":\"enum1\",\n" + + " \"type\":{\"type\":\"enum\", \"name\":\"enum1_values\", " + + "\"symbols\":[\"ENUM1_VALUES_VALUE1\",\"ENUM1_VALUES_VALUE2\", \"ENUM1_VALUES_VALUE3\"]}\n" + + " },\n" + + " {\n" + + " \"name\":\"array1\",\n" + + " \"type\":{\"type\":\"array\", \"items\":\"string\"}\n" + + " },\n" + + " {\n" + + " \"name\":\"map1\",\n" + + " \"type\":{\"type\":\"map\", \"values\":\"string\"}\n" + + " },\n" + + " {\n" + + " \"name\":\"union1\",\n" + + " \"type\":[\"float\", \"boolean\", \"string\"]\n" + + " },\n" + + " {\n" + + " \"name\":\"fixed1\",\n" + + " \"type\":{\"type\":\"fixed\", \"name\":\"fourbytes\", \"size\":4}\n" + + " },\n" + + " {\n" + + " \"name\":\"null1\",\n" + + " \"type\":\"null\"\n" + + " },\n" + + " {\n" + + " \"name\":\"UnionNullInt\",\n" + + " \"type\":[\"int\", \"null\"]\n" + + " },\n" + + " {\n" + + " \"name\":\"bytes1\",\n" + + " \"type\":\"bytes\"\n" + + " }\n" + + " ]\n" + + "}"; + + @Before + public void setUp() throws Exception { + conf = MetastoreConf.newMetastoreConf(); + warehouse = new Warehouse(conf); + + // set some values to use for getting conf. vars + MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.METRICS_ENABLED, true); + conf.set("hive.key1", "value1"); + conf.set("hive.key2", "http://www.example.com"); + conf.set("hive.key3", ""); + conf.set("hive.key4", "0"); + conf.set("datanucleus.autoCreateTables", "false"); + + MetaStoreTestUtils.setConfForStandloneMode(conf); + MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX, 2); + MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.LIMIT_PARTITION_REQUEST, + DEFAULT_LIMIT_PARTITION_REQUEST); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL, + DefaultStorageSchemaReader.class.getName()); + client = createClient(); + } + + @After + public void closeClient() { + client.close(); + } + + private void silentDropDatabase(String dbName) throws TException { + try { + for (String tableName : client.getTables(dbName, "*")) { + client.dropTable(dbName, tableName); + } + client.dropDatabase(dbName); + } catch (NoSuchObjectException | InvalidOperationException e) { + // NOP + } + } + + private HiveMetaStoreClient createClient() throws Exception { + try { + return new HiveMetaStoreClient(conf); + } catch (Throwable e) { + System.err.println("Unable to open the metastore"); + System.err.println(StringUtils.stringifyException(e)); + throw new Exception(e); + } + } + + @Test + public void testSimpleAvroTable() throws TException, IOException { + List fields = new ArrayList<>(2); + FieldSchema field = new FieldSchema(); + field.setName("name"); + field.setType("string"); + field.setComment("Test name comment"); + fields.add(field); + + field = new FieldSchema(); + field.setName("age"); + field.setType("int"); + field.setComment("Test age comment"); + fields.add(field); + + createTable(TEST_DB_NAME, TEST_TABLE_NAME, AVRO_SERIALIZATION_LIB, fields, null); + List retFields = client.getFields("testdb", "testTbl"); + verifyTableFields(fields, retFields, null); + } + + private Table createTable(String dbName, String tblName, String serializationLib, + List fields, Map tblProperties) throws TException, IOException { + client.dropTable(dbName, tblName); + silentDropDatabase(dbName); + Database db = new Database(); + db.setName(dbName); + client.createDatabase(db); + db = client.getDatabase(dbName); + Path dbPath = new Path(db.getLocationUri()); + FileSystem fs = FileSystem.get(dbPath.toUri(), conf); + String typeName = "dummy"; + client.dropType(typeName); + Type typ1 = new Type(); + typ1.setName(typeName); + typ1.setFields(fields); + client.createType(typ1); + + Table tbl = new TableBuilder().setDbName(dbName).setTableName(tblName).setCols(typ1.getFields()) + .setSerdeLib(serializationLib).setTableParams(tblProperties).build(); + client.createTable(tbl); + return client.getTable(dbName, tblName); + } + + @Test + public void testExternalSchemaAvroTable() throws TException, IOException { + //map + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, MAP_WITH_PRIMITIVE_VALUE_TYPE); + List retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "aMap", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "map", + retFields.get(0).getType()); + Assert.assertEquals("Unexpected comment of the field", "", retFields.get(0).getComment()); + + //list + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, + ARRAY_WITH_PRIMITIVE_ELEMENT_TYPE); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "anArray", retFields.get(0).getName()); + Assert + .assertEquals("Unexpected type of the field", "array", retFields.get(0).getType()); + + //struct + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, RECORD_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "aRecord", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", + "struct", retFields.get(0).getType()); + + //union + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, UNION_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "aUnion", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "uniontype", + retFields.get(0).getType()); + + //union-2 + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, UNION_SCHEMA_2); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "aUnion", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "uniontype", + retFields.get(0).getType()); + + //union_3 + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, UNION_SCHEMA_3); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "aUnion", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "uniontype", + retFields.get(0).getType()); + + //union_4 + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, UNION_SCHEMA_4); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "aUnion", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "uniontype", + retFields.get(0).getType()); + + //enum + // Enums are one of two Avro types that Hive doesn't have any native support for. + // Column names - we lose the enumness of this schema + // Column types become string + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, ENUM_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "baddies", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "string", + retFields.get(0).getType()); + + // Hive has no concept of Avro's fixed type. Fixed -> arrays of bytes + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, FIXED_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "hash", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "binary", + retFields.get(0).getType()); + + //nullable string + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, NULLABLE_STRING_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "nullableString", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "string", + retFields.get(0).getType()); + + //map with nullable value - That Union[T, NULL] is converted to just T, within a Map + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "aMap", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "map", + retFields.get(0).getType()); + + // That Union[T, NULL] is converted to just T. + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, NULLABLE_ENUM_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "nullableEnum", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "string", + retFields.get(0).getType()); + + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, BYTES_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "bytesField", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "binary", + retFields.get(0).getType()); + + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, KITCHEN_SINK_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 16, retFields.size()); + //There are 16 fields in this schema. Instead of verifying all we verify the interesting ones + //(ones which have not been tested above) + Assert + .assertEquals("Unexpected name of 8th field", "inner_record1", retFields.get(7).getName()); + Assert.assertEquals("Unexpected type of the field", + "struct", + retFields.get(7).getType()); + + Assert.assertEquals("Unexpected field name of the 10th field", "array1", + retFields.get(9).getName()); + Assert.assertEquals("Unexpected field type of the 10th field", "array", + retFields.get(9).getType()); + + Assert.assertEquals("Unexpected field name of the 11th field", "map1", + retFields.get(10).getName()); + Assert.assertEquals("Unexpected field type of the 11th field", "map", + retFields.get(10).getType()); + + Assert.assertEquals("Unexpected field name of the 12th field", "union1", + retFields.get(11).getName()); + Assert + .assertEquals("Unexpected field type of the 12th field", "uniontype", + retFields.get(11).getType()); + + Assert.assertEquals("Unexpected field name of the 14th field", "null1", + retFields.get(13).getName()); + Assert.assertEquals("Unexpected field type of the 14th field", "void", + retFields.get(13).getType()); + + Assert.assertEquals("Unexpected field name of the 15th field", "UnionNullInt", + retFields.get(14).getName()); + Assert.assertEquals("Unexpected field type of the 15th field", "int", + retFields.get(14).getType()); + } + + private void createAvroTableWithExternalSchema(String dbName, String tblName, String schemaStr) + throws TException, IOException { + List fields = new ArrayList<>(0); + Map tblParams = new HashMap<>(); + tblParams.put(AvroSchemaUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName(), schemaStr); + createTable(dbName, tblName, AVRO_SERIALIZATION_LIB, fields, tblParams); + } + + @Test + public void testSimpleTable() throws TException, IOException { + List fields = new ArrayList<>(2); + FieldSchema field = new FieldSchema(); + field.setName("name"); + field.setType("string"); + field.setComment("Test name comment"); + fields.add(field); + + field = new FieldSchema(); + field.setName("age"); + field.setType("int"); + field.setComment("Test age comment"); + fields.add(field); + + createTable(TEST_DB_NAME, TEST_TABLE_NAME, null, fields, null); + List retFields = client.getFields("testDb", "testTbl"); + verifyTableFields(fields, retFields, null); + } + + private void verifyTableFields(List expected, List actual, + String nullCommentText) { + Assert.assertEquals(expected.size(), actual.size()); + int size = expected.size(); + for (int i = 0; i < size; i++) { + FieldSchema expectedField = expected.get(i); + FieldSchema actualField = actual.get(i); + Assert.assertEquals("Name does not match for field " + (i + 1), expectedField.getName(), + actualField.getName()); + Assert.assertEquals("Type does not match for field " + (i + 1), expectedField.getType(), + actualField.getType()); + String expectedComment = null; + if (expectedField.getComment() == null && nullCommentText != null) { + expectedComment = nullCommentText; + } else { + expectedComment = expectedField.getComment(); + } + Assert.assertEquals("Comment does not match for field " + (i + 1), expectedComment, + actualField.getComment()); + } + } +} diff --git a/storage-api/hive-storage-api/pom.xml b/storage-api/hive-storage-api/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..f6680687576d28969dfad983a81f432625d945fe --- /dev/null +++ b/storage-api/hive-storage-api/pom.xml @@ -0,0 +1,133 @@ + + + + 4.0.0 + + org.apache.hive + storage-api + 3.0.0-SNAPSHOT + ../pom.xml + + + org.apache.hive + hive-storage-api + 3.0.0-SNAPSHOT + jar + Hive Storage API + + + 2.6 + 1.1.3 + 14.0.1 + 2.8.1 + 4.11 + 1.7.10 + 3.0.0 + + + + + + commons-lang + commons-lang + ${commons-lang.version} + + + org.apache.hadoop + hadoop-annotations + ${hadoop.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + com.google.guava + guava + + + com.google.code.findbugs + jsr305 + + + commons-logging + commons-logging + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + org.apache.avro + avro + + + org.apache.httpcomponents + httpcore + + + org.apache.httpcomponents + httpclient + + + org.slf4j + slf4j-log4j12 + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + + com.google.guava + guava + ${guava.version} + test + + + junit + junit + ${junit.version} + test + + + commons-logging + commons-logging + ${commons-logging.version} + test + + + + diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..e5025bf1407aaf51910849d70e4ccda08a70964d --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hive.common.io.DiskRange; + +/** + * Disk range information class containing disk ranges and total length. + */ +public class DiskRangeInfo { + List diskRanges; // TODO: use DiskRangeList instead + long totalLength; + + public DiskRangeInfo(int indexBaseOffset) { + this.diskRanges = new ArrayList<>(); + // Some data is missing from the stream for PPD uncompressed read (because index offset is + // relative to the entire stream and we only read part of stream if RGs are filtered; unlike + // with compressed data where PPD only filters CBs, so we always get full CB, and index offset + // is relative to CB). To take care of the case when UncompressedStream goes seeking around by + // its incorrect (relative to partial stream) index offset, we will increase the length by our + // offset-relative-to-the-stream, and also account for it in buffers (see createDiskRangeInfo). + // So, index offset now works; as long as noone seeks into this data before the RG (why would + // they), everything works. This is hacky... Stream shouldn't depend on having all the data. + this.totalLength = indexBaseOffset; + } + + public void addDiskRange(DiskRange diskRange) { + diskRanges.add(diskRange); + totalLength += diskRange.getLength(); + } + + public List getDiskRanges() { + return diskRanges; + } + + public long getTotalLength() { + return totalLength; + } +} + diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/Pool.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/Pool.java new file mode 100644 index 0000000000000000000000000000000000000000..5ab4ee7802fc385ff04e0f4fcd54398ed31073c5 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/Pool.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common; + +/** Simple object pool to prevent GC on small objects passed between threads. */ +public interface Pool { + /** Object helper for objects stored in the pool. */ + public interface PoolObjectHelper { + /** Called to create an object when one cannot be provided. + * @return a newly allocated object + */ + T create(); + /** Called before the object is put in the pool (regardless of whether put succeeds). + * @param t the object to reset + */ + void resetBeforeOffer(T t); + } + + T take(); + void offer(T t); + int size(); +} \ No newline at end of file diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java new file mode 100644 index 0000000000000000000000000000000000000000..94b8c58d28219c13f045f6d9e8dd093a4b3607d9 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common; + +import java.util.Arrays; +import java.util.BitSet; + +/** + * An implementation of {@link org.apache.hadoop.hive.common.ValidTxnList} for use by the compactor. + * + * Compaction should only include txns up to smallest open txn (exclussive). + * There may be aborted txns in the snapshot represented by this ValidCompactorTxnList. + * Thus {@link #isTxnRangeValid(long, long)} returns NONE for any range that inluces any unresolved + * transactions. Any txn above {@code highWatermark} is unresolved. + * These produce the logic we need to assure that the compactor only sees records less than the lowest + * open transaction when choosing which files to compact, but that it still ignores aborted + * records when compacting. + * + * See org.apache.hadoop.hive.metastore.txn.TxnUtils#createValidCompactTxnList() for proper + * way to construct this. + */ +public class ValidCompactorTxnList extends ValidReadTxnList { + public ValidCompactorTxnList() { + super(); + } + public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark) { + this(abortedTxnList, abortedBits, highWatermark, Long.MAX_VALUE); + } + /** + * @param abortedTxnList list of all aborted transactions + * @param abortedBits bitset marking whether the corresponding transaction is aborted + * @param highWatermark highest committed transaction to be considered for compaction, + * equivalently (lowest_open_txn - 1). + */ + public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark, long minOpenTxnId) { + // abortedBits should be all true as everything in exceptions are aborted txns + super(abortedTxnList, abortedBits, highWatermark, minOpenTxnId); + if(this.exceptions.length <= 0) { + return; + } + //now that exceptions (aka abortedTxnList) is sorted + int idx = Arrays.binarySearch(this.exceptions, highWatermark); + int lastElementPos; + if(idx < 0) { + int insertionPoint = -idx - 1 ;//see Arrays.binarySearch() JavaDoc + lastElementPos = insertionPoint - 1; + } + else { + lastElementPos = idx; + } + /* + * ensure that we throw out any exceptions above highWatermark to make + * {@link #isTxnValid(long)} faster + */ + this.exceptions = Arrays.copyOf(this.exceptions, lastElementPos + 1); + } + public ValidCompactorTxnList(String value) { + super(value); + } + /** + * Returns org.apache.hadoop.hive.common.ValidTxnList.RangeResponse.ALL if all txns in + * the range are resolved and RangeResponse.NONE otherwise + */ + @Override + public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId) { + return highWatermark >= maxTxnId ? RangeResponse.ALL : RangeResponse.NONE; + } + + @Override + public boolean isTxnAborted(long txnid) { + return Arrays.binarySearch(exceptions, txnid) >= 0; + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java new file mode 100644 index 0000000000000000000000000000000000000000..ccdd4b727a59b106da0def2d0173c917e1c5c0f7 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common; + +import java.util.Arrays; +import java.util.BitSet; + +/** + * An implementation of {@link org.apache.hadoop.hive.common.ValidTxnList} for use by readers. + * This class will view a transaction as valid only if it is committed. Both open and aborted + * transactions will be seen as invalid. + */ +public class ValidReadTxnList implements ValidTxnList { + + protected long[] exceptions; + protected BitSet abortedBits; // BitSet for flagging aborted transactions. Bit is true if aborted, false if open + //default value means there are no open txn in the snapshot + private long minOpenTxn = Long.MAX_VALUE; + protected long highWatermark; + + public ValidReadTxnList() { + this(new long[0], new BitSet(), Long.MAX_VALUE, Long.MAX_VALUE); + } + + /** + * Used if there are no open transactions in the snapshot + */ + public ValidReadTxnList(long[] exceptions, BitSet abortedBits, long highWatermark) { + this(exceptions, abortedBits, highWatermark, Long.MAX_VALUE); + } + public ValidReadTxnList(long[] exceptions, BitSet abortedBits, long highWatermark, long minOpenTxn) { + if (exceptions.length > 0) { + this.minOpenTxn = minOpenTxn; + } + this.exceptions = exceptions; + this.abortedBits = abortedBits; + this.highWatermark = highWatermark; + } + + public ValidReadTxnList(String value) { + readFromString(value); + } + + @Override + public boolean isTxnValid(long txnid) { + if (highWatermark < txnid) { + return false; + } + return Arrays.binarySearch(exceptions, txnid) < 0; + } + + /** + * We cannot use a base file if its range contains an open txn. + * @param txnid from base_xxxx + */ + @Override + public boolean isValidBase(long txnid) { + return minOpenTxn > txnid && txnid <= highWatermark; + } + @Override + public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId) { + // check the easy cases first + if (highWatermark < minTxnId) { + return RangeResponse.NONE; + } else if (exceptions.length > 0 && exceptions[0] > maxTxnId) { + return RangeResponse.ALL; + } + + // since the exceptions and the range in question overlap, count the + // exceptions in the range + long count = Math.max(0, maxTxnId - highWatermark); + for(long txn: exceptions) { + if (minTxnId <= txn && txn <= maxTxnId) { + count += 1; + } + } + + if (count == 0) { + return RangeResponse.ALL; + } else if (count == (maxTxnId - minTxnId + 1)) { + return RangeResponse.NONE; + } else { + return RangeResponse.SOME; + } + } + + @Override + public String toString() { + return writeToString(); + } + + @Override + public String writeToString() { + StringBuilder buf = new StringBuilder(); + buf.append(highWatermark); + buf.append(':'); + buf.append(minOpenTxn); + if (exceptions.length == 0) { + buf.append(':'); // separator for open txns + buf.append(':'); // separator for aborted txns + } else { + StringBuilder open = new StringBuilder(); + StringBuilder abort = new StringBuilder(); + for (int i = 0; i < exceptions.length; i++) { + if (abortedBits.get(i)) { + if (abort.length() > 0) { + abort.append(','); + } + abort.append(exceptions[i]); + } else { + if (open.length() > 0) { + open.append(','); + } + open.append(exceptions[i]); + } + } + buf.append(':'); + buf.append(open); + buf.append(':'); + buf.append(abort); + } + return buf.toString(); + } + + @Override + public void readFromString(String src) { + if (src == null || src.length() == 0) { + highWatermark = Long.MAX_VALUE; + exceptions = new long[0]; + abortedBits = new BitSet(); + } else { + String[] values = src.split(":"); + highWatermark = Long.parseLong(values[0]); + minOpenTxn = Long.parseLong(values[1]); + String[] openTxns = new String[0]; + String[] abortedTxns = new String[0]; + if (values.length < 3) { + openTxns = new String[0]; + abortedTxns = new String[0]; + } else if (values.length == 3) { + if (!values[2].isEmpty()) { + openTxns = values[2].split(","); + } + } else { + if (!values[2].isEmpty()) { + openTxns = values[2].split(","); + } + if (!values[3].isEmpty()) { + abortedTxns = values[3].split(","); + } + } + exceptions = new long[openTxns.length + abortedTxns.length]; + int i = 0; + for (String open : openTxns) { + exceptions[i++] = Long.parseLong(open); + } + for (String abort : abortedTxns) { + exceptions[i++] = Long.parseLong(abort); + } + Arrays.sort(exceptions); + abortedBits = new BitSet(exceptions.length); + for (String abort : abortedTxns) { + int index = Arrays.binarySearch(exceptions, Long.parseLong(abort)); + abortedBits.set(index); + } + } + } + + @Override + public long getHighWatermark() { + return highWatermark; + } + + @Override + public long[] getInvalidTransactions() { + return exceptions; + } + + @Override + public Long getMinOpenTxn() { + return minOpenTxn == Long.MAX_VALUE ? null : minOpenTxn; + } + + @Override + public boolean isTxnAborted(long txnid) { + int index = Arrays.binarySearch(exceptions, txnid); + return index >= 0 && abortedBits.get(index); + } + + @Override + public RangeResponse isTxnRangeAborted(long minTxnId, long maxTxnId) { + // check the easy cases first + if (highWatermark < minTxnId) { + return RangeResponse.NONE; + } + + int count = 0; // number of aborted txns found in exceptions + + // traverse the aborted txns list, starting at first aborted txn index + for (int i = abortedBits.nextSetBit(0); i >= 0; i = abortedBits.nextSetBit(i + 1)) { + long abortedTxnId = exceptions[i]; + if (abortedTxnId > maxTxnId) { // we've already gone beyond the specified range + break; + } + if (abortedTxnId >= minTxnId && abortedTxnId <= maxTxnId) { + count++; + } + } + + if (count == 0) { + return RangeResponse.NONE; + } else if (count == (maxTxnId - minTxnId + 1)) { + return RangeResponse.ALL; + } else { + return RangeResponse.SOME; + } + } +} + diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java new file mode 100644 index 0000000000000000000000000000000000000000..108e5ca85cfab26403e74adefd7f349d4105ee6b --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common; + +/** + * Models the list of transactions that should be included in a snapshot. + * It is modelled as a high water mark, which is the largest transaction id that + * has been committed and a list of transactions that are not included. + */ +public interface ValidTxnList { + + /** + * Key used to store valid txn list in a + * {@link org.apache.hadoop.conf.Configuration} object. + */ + public static final String VALID_TXNS_KEY = "hive.txn.valid.txns"; + + /** + * The response to a range query. NONE means no values in this range match, + * SOME mean that some do, and ALL means that every value does. + */ + public enum RangeResponse {NONE, SOME, ALL}; + + /** + * Indicates whether a given transaction is valid. Note that valid may have different meanings + * for different implementations, as some will only want to see committed transactions and some + * both committed and aborted. + * @param txnid id for the transaction + * @return true if valid, false otherwise + */ + public boolean isTxnValid(long txnid); + + /** + * Returns {@code true} if such base file can be used to materialize the snapshot represented by + * this {@code ValidTxnList}. + * @param txnid highest txn in a given base_xxxx file + */ + public boolean isValidBase(long txnid); + + /** + * Find out if a range of transaction ids are valid. Note that valid may have different meanings + * for different implementations, as some will only want to see committed transactions and some + * both committed and aborted. + * @param minTxnId minimum txnid to look for, inclusive + * @param maxTxnId maximum txnid to look for, inclusive + * @return Indicate whether none, some, or all of these transactions are valid. + */ + public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId); + + /** + * Write this validTxnList into a string. This should produce a string that + * can be used by {@link #readFromString(String)} to populate a validTxnsList. + */ + public String writeToString(); + + /** + * Populate this validTxnList from the string. It is assumed that the string + * was created via {@link #writeToString()} and the exceptions list is sorted. + * @param src source string. + */ + public void readFromString(String src); + + /** + * Get the largest transaction id used. + * @return largest transaction id used + */ + public long getHighWatermark(); + + /** + * Get the list of transactions under the high water mark that are not valid. Note that invalid + * may have different meanings for different implementations, as some will only want to see open + * transactions and some both open and aborted. + * @return a list of invalid transaction ids + */ + public long[] getInvalidTransactions(); + + /** + * Indicates whether a given transaction is aborted. + * @param txnid id for the transaction + * @return true if aborted, false otherwise + */ + public boolean isTxnAborted(long txnid); + + /** + * Find out if a range of transaction ids are aborted. + * @param minTxnId minimum txnid to look for, inclusive + * @param maxTxnId maximum txnid to look for, inclusive + * @return Indicate whether none, some, or all of these transactions are aborted. + */ + public RangeResponse isTxnRangeAborted(long minTxnId, long maxTxnId); + + /** + * Returns smallest Open transaction in this set, {@code null} if there is none. + */ + Long getMinOpenTxn(); +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java new file mode 100644 index 0000000000000000000000000000000000000000..775233cb6e3cd091b570634a7e030e4352f24ea6 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.io; + +import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer; + +/** An allocator provided externally to storage classes to allocate MemoryBuffer-s. */ +public interface Allocator { + public static class AllocatorOutOfMemoryException extends RuntimeException { + public AllocatorOutOfMemoryException(String msg) { + super(msg); + } + + private static final long serialVersionUID = 268124648177151761L; + } + + public interface BufferObjectFactory { + MemoryBuffer create(); + } + + /** + * Allocates multiple buffers of a given size. + * @param dest Array where buffers are placed. Objects are reused if already there + * (see createUnallocated), created otherwise. + * @param size Allocation size. + * @throws AllocatorOutOfMemoryException Cannot allocate. + */ + @Deprecated + void allocateMultiple(MemoryBuffer[] dest, int size) throws AllocatorOutOfMemoryException; + + /** + * Allocates multiple buffers of a given size. + * @param dest Array where buffers are placed. Objects are reused if already there + * (see createUnallocated), created otherwise. + * @param size Allocation size. + * @param factory A factory to create the objects in the dest array, if needed. + * @throws AllocatorOutOfMemoryException Cannot allocate. + */ + void allocateMultiple(MemoryBuffer[] dest, int size, BufferObjectFactory factory) + throws AllocatorOutOfMemoryException; + + /** + * Creates an unallocated memory buffer object. This object can be passed to allocateMultiple + * to allocate; this is useful if data structures are created for separate buffers that can + * later be allocated together. + * @return a new unallocated memory buffer + */ + @Deprecated + MemoryBuffer createUnallocated(); + + /** Deallocates a memory buffer. + * @param buffer the buffer to deallocate + */ + void deallocate(MemoryBuffer buffer); + + /** Whether the allocator uses direct buffers. + * @return are they direct buffers? + */ + boolean isDirectAlloc(); + + /** Maximum allocation size supported by this allocator. + * @return a number of bytes + */ + int getMaxAllocation(); +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java new file mode 100644 index 0000000000000000000000000000000000000000..552f20e276a6cd2d93ecc09292985c842ee22b87 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.io; + +import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer; + +/** An abstract data cache that IO formats can use to retrieve and cache data. */ +public interface DataCache { + public static final class BooleanRef { + public boolean value; + } + + /** Disk range factory used during cache retrieval. */ + public interface DiskRangeListFactory { + DiskRangeList createCacheChunk(MemoryBuffer buffer, long startOffset, long endOffset); + } + + /** + * Gets file data for particular offsets. The range list is modified in place; it is then + * returned (since the list head could have changed). Ranges are replaced with cached ranges. + * + * Any such buffer is locked in cache to prevent eviction, and must therefore be released + * back to cache via a corresponding call (releaseBuffer) when the caller is done with it. + * + * In case of partial overlap with cached data, full cache blocks are always returned; + * there's no capacity for partial matches in return type. The rules are as follows: + * 1) If the requested range starts in the middle of a cached range, that cached range will not + * be returned by default (e.g. if [100,200) and [200,300) are cached, the request for + * [150,300) will only return [200,300) from cache). This may be configurable in impls. + * This is because we assume well-known range start offsets are used (rg/stripe offsets), so + * a request from the middle of the start doesn't make sense. + * 2) If the requested range ends in the middle of a cached range, that entire cached range will + * be returned (e.g. if [100,200) and [200,300) are cached, the request for [100,250) will + * return both ranges). It should really be same as #1, however currently ORC uses estimated + * end offsets; if we don't return the end block, the caller may read it from disk needlessly. + * + * @param fileKey Unique ID of the target file on the file system. + * @param range A set of DiskRange-s (linked list) that is to be retrieved. May be modified. + * @param baseOffset base offset for the ranges (stripe/stream offset in case of ORC). + * @param factory A factory to produce DiskRangeList-s out of cached MemoryBuffer-s. + * @param gotAllData An out param - whether all the requested data was found in cache. + * @return The new or modified list of DiskRange-s, where some ranges may contain cached data. + */ + DiskRangeList getFileData(Object fileKey, DiskRangeList range, long baseOffset, + DiskRangeListFactory factory, BooleanRef gotAllData); + + /** + * Puts file data into cache, or gets older data in case of collisions. + * + * The memory buffers provided MUST be allocated via an allocator returned by getAllocator + * method, to allow cache implementations that evict and then de-allocate the buffer. + * + * It is assumed that the caller will use the data immediately, therefore any buffers provided + * to putFileData (or returned due to cache collision) are locked in cache to prevent eviction, + * and must therefore be released back to cache via a corresponding call (releaseBuffer) when the + * caller is done with it. Buffers rejected due to conflict will neither be locked, nor + * automatically deallocated. The caller must take care to discard these buffers. + * + * @param fileKey Unique ID of the target file on the file system. + * @param ranges The ranges for which the data is being cached. These objects will not be stored. + * @param data The data for the corresponding ranges. + * @param baseOffset base offset for the ranges (stripe/stream offset in case of ORC). + * @return null if all data was put; bitmask indicating which chunks were not put otherwise; + * the replacement chunks from cache are updated directly in the array. + */ + long[] putFileData(Object fileKey, DiskRange[] ranges, MemoryBuffer[] data, long baseOffset); + + /** + * Releases the buffer returned by getFileData/provided to putFileData back to cache. + * See respective javadocs for details. + * @param buffer the buffer to release + */ + void releaseBuffer(MemoryBuffer buffer); + + /** + * Notifies the cache that the buffer returned from getFileData/provided to putFileData will + * be used by another consumer and therefore released multiple times (one more time per call). + * @param buffer the buffer to reuse + */ + void reuseBuffer(MemoryBuffer buffer); + + /** + * Gets the allocator associated with this DataCache. + * @return the allocator + */ + Allocator getAllocator(); + + /** + * Gets the buffer object factory associated with this DataCache, to use with allocator. + * @return the factory + */ + Allocator.BufferObjectFactory getDataBufferFactory(); +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRange.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRange.java new file mode 100644 index 0000000000000000000000000000000000000000..33aecf53e703b72482e4e29c19170bbdebc815e3 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRange.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.io; + +import java.nio.ByteBuffer; + +/** + * The sections of a file. + */ +public class DiskRange { + /** The first address. */ + protected long offset; + /** The address afterwards. */ + protected long end; + + public DiskRange(long offset, long end) { + this.offset = offset; + this.end = end; + if (end < offset) { + throw new IllegalArgumentException("invalid range " + this); + } + } + + @Override + public boolean equals(Object other) { + if (other == null || other.getClass() != getClass()) { + return false; + } + return equalRange((DiskRange) other); + } + + public boolean equalRange(DiskRange other) { + return other.offset == offset && other.end == end; + } + + @Override + public int hashCode() { + return (int)(offset ^ (offset >>> 32)) * 31 + (int)(end ^ (end >>> 32)); + } + + @Override + public String toString() { + return "range start: " + offset + " end: " + end; + } + + public long getOffset() { + return offset; + } + + public long getEnd() { + return end; + } + + public int getLength() { + long len = this.end - this.offset; + assert len <= Integer.MAX_VALUE; + return (int)len; + } + + // For subclasses + public boolean hasData() { + return false; + } + + public DiskRange sliceAndShift(long offset, long end, long shiftBy) { + // Rather, unexpected usage exception. + throw new UnsupportedOperationException(); + } + + public ByteBuffer getData() { + throw new UnsupportedOperationException(); + } + + protected boolean merge(long otherOffset, long otherEnd) { + if (!overlap(offset, end, otherOffset, otherEnd)) return false; + offset = Math.min(offset, otherOffset); + end = Math.max(end, otherEnd); + return true; + } + + private static boolean overlap(long leftA, long rightA, long leftB, long rightB) { + if (leftA <= leftB) { + return rightA >= leftB; + } + return rightB >= leftA; + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java new file mode 100644 index 0000000000000000000000000000000000000000..13494b6e413aff1ba4c5a97bd3afa96e39d148a7 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java @@ -0,0 +1,297 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.io; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Java linked list iterator interface is convoluted, and moreover concurrent modifications + * of the same list by multiple iterators are impossible. Hence, this. + * Java also doesn't support multiple inheritance, so this cannot be done as "aspect"... */ +public class DiskRangeList extends DiskRange { + private static final Logger LOG = LoggerFactory.getLogger(DiskRangeList.class); + public DiskRangeList prev, next; + + public DiskRangeList(long offset, long end) { + super(offset, end); + } + + /** Replaces this element with another in the list; returns the new element. + * @param other the disk range to swap into this list + * @return the new element + */ + public DiskRangeList replaceSelfWith(DiskRangeList other) { + checkArg(other); + other.prev = this.prev; + other.next = this.next; + if (this.prev != null) { + checkOrder(this.prev, other, this); + this.prev.next = other; + } + if (this.next != null) { + checkOrder(other, this.next, this); + this.next.prev = other; + } + this.next = this.prev = null; + return other; + } + + private final static void checkOrder(DiskRangeList prev, DiskRangeList next, DiskRangeList ref) { + if (prev.getEnd() <= next.getOffset()) return; + assertInvalidOrder(ref.prev == null ? ref : ref.prev, prev, next); + } + + private final static void assertInvalidOrder( + DiskRangeList ref, DiskRangeList prev, DiskRangeList next) { + String error = "Elements not in order " + prev + " and " + next + + "; trying to insert into " + stringifyDiskRanges(ref); + LOG.error(error); + throw new AssertionError(error); + } + + // TODO: this duplicates a method in ORC, but the method should actually be here. + public final static String stringifyDiskRanges(DiskRangeList range) { + StringBuilder buffer = new StringBuilder(); + buffer.append("["); + boolean isFirst = true; + while (range != null) { + if (!isFirst) { + buffer.append(", {"); + } else { + buffer.append("{"); + } + isFirst = false; + buffer.append(range.toString()); + buffer.append("}"); + range = range.next; + } + buffer.append("]"); + return buffer.toString(); + } + + private void checkArg(DiskRangeList other) { + if (other == this) { + // The only case where duplicate elements matter... the others are handled by the below. + throw new AssertionError("Inserting self into the list [" + other + "]"); + } + if (other.prev != null || other.next != null) { + throw new AssertionError("[" + other + "] is part of another list; prev [" + + other.prev + "], next [" + other.next + "]"); + } + } + + /** + * Inserts an intersecting range before current in the list and adjusts offset accordingly. + * @param other the element to insert + * @return the new element. + */ + public DiskRangeList insertPartBefore(DiskRangeList other) { + checkArg(other); + if (other.end <= this.offset || other.end > this.end) { + assertInvalidOrder(this.prev == null ? this : this.prev, other, this); + } + this.offset = other.end; + other.prev = this.prev; + other.next = this; + if (this.prev != null) { + checkOrder(this.prev, other, this.prev); + this.prev.next = other; + } + this.prev = other; + return other; + } + + /** + * Inserts an element after current in the list. + * @param other the new element to insert + * @return the new element. + * */ + public DiskRangeList insertAfter(DiskRangeList other) { + checkArg(other); + checkOrder(this, other, this); + return insertAfterInternal(other); + } + + private DiskRangeList insertAfterInternal(DiskRangeList other) { + other.next = this.next; + other.prev = this; + if (this.next != null) { + checkOrder(other, this.next, this); + this.next.prev = other; + } + this.next = other; + return other; + } + + /** + * Inserts an intersecting range after current in the list and adjusts offset accordingly. + * @param other the new element to insert + * @return the new element. + */ + public DiskRangeList insertPartAfter(DiskRangeList other) { + // The only allowed non-overlapping option is extra bytes at the end. + if (other.offset > this.end || other.offset <= this.offset || other.end <= this.offset) { + assertInvalidOrder(this.prev == null ? this : this.prev, this, other); + } + this.end = other.offset; + return insertAfter(other); + } + + /** Removes an element after current from the list. */ + public void removeAfter() { + DiskRangeList other = this.next; + if (this == other) { + throw new AssertionError("Invalid duplicate [" + other + "]"); + } + this.next = other.next; + if (this.next != null) { + this.next.prev = this; + } + other.next = other.prev = null; + } + + /** Removes the current element from the list. */ + public void removeSelf() { + if (this.prev == this || this.next == this) { + throw new AssertionError("Invalid duplicate [" + this + "]"); + } + if (this.prev != null) { + this.prev.next = this.next; + } + if (this.next != null) { + this.next.prev = this.prev; + } + this.next = this.prev = null; + } + + /** Splits current element in the list, using DiskRange::slice. + * @param cOffset the position to split the list + * @return the split list + */ + public final DiskRangeList split(long cOffset) { + DiskRangeList right = insertAfterInternal((DiskRangeList)this.sliceAndShift(cOffset, end, 0)); + DiskRangeList left = replaceSelfWith((DiskRangeList)this.sliceAndShift(offset, cOffset, 0)); + checkOrder(left, right, left); // Prev/next are already checked in the calls. + return left; + } + + public boolean hasContiguousNext() { + return next != null && end == next.offset; + } + + // @VisibleForTesting + public int listSize() { + int result = 1; + DiskRangeList current = this.next; + while (current != null) { + ++result; + current = current.next; + } + return result; + } + + public long getTotalLength() { + long totalLength = getLength(); + DiskRangeList current = next; + while (current != null) { + totalLength += current.getLength(); + current = current.next; + } + return totalLength; + } + + // @VisibleForTesting + public DiskRangeList[] listToArray() { + DiskRangeList[] result = new DiskRangeList[listSize()]; + int i = 0; + DiskRangeList current = this.next; + while (current != null) { + result[i] = current; + ++i; + current = current.next; + } + return result; + } + + public static class CreateHelper { + private DiskRangeList tail = null, head; + + public DiskRangeList getTail() { + return tail; + } + + public void addOrMerge(long offset, long end, boolean doMerge, boolean doLogNew) { + if (doMerge && tail != null && tail.merge(offset, end)) return; + if (doLogNew) { + LOG.debug("Creating new range; last range (which can include some previous adds) was " + + tail); + } + DiskRangeList node = new DiskRangeList(offset, end); + if (tail == null) { + head = tail = node; + } else { + tail = tail.insertAfter(node); + } + } + + public DiskRangeList get() { + return head; + } + + public DiskRangeList extract() { + DiskRangeList result = head; + head = null; + return result; + } + } + + /** + * List in-place mutation helper - a bogus first element that is inserted before list head, + * and thus remains constant even if head is replaced with some new range via in-place list + * mutation. extract() can be used to obtain the modified list. + */ + public static class MutateHelper extends DiskRangeList { + public MutateHelper(DiskRangeList head) { + super(-1, -1); + assert head != null; + assert head.prev == null; + this.next = head; + head.prev = this; + } + + public DiskRangeList get() { + return next; + } + + public DiskRangeList extract() { + DiskRangeList result = this.next; + assert result != null; + this.next = result.prev = null; + return result; + } + } + + public void setEnd(long newEnd) { + assert newEnd >= this.offset; + assert this.next == null || this.next.offset >= newEnd; + this.end = newEnd; + if (this.next != null) { + checkOrder(this, this.next, this); + } + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java new file mode 100644 index 0000000000000000000000000000000000000000..e684ece3edbf420e05adadc7a1f9599553ceb6d4 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.io; + + +import java.nio.ByteBuffer; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.hadoop.hive.common.io.encoded.MemoryBufferOrBuffers; + +public interface FileMetadataCache { + /** + * @return Metadata for a given file (ORC or Parquet footer). + * The caller must decref this buffer when done. + */ + MemoryBufferOrBuffers getFileMetadata(Object fileKey); + + /** + * Puts the metadata for a given file (e.g. a footer buffer into cache). + * @param fileKey The file key. + * @param length The footer length. + * @param is The stream to read the footer from. + * @return The buffer or buffers representing the cached footer. + * The caller must decref this buffer when done. + */ + MemoryBufferOrBuffers putFileMetadata( + Object fileKey, int length, InputStream is) throws IOException; + + MemoryBufferOrBuffers putFileMetadata(Object fileKey, ByteBuffer tailBuffer); + + /** + * Releases the buffer returned from getFileMetadata or putFileMetadata method. + * @param buffer The buffer to release. + */ + void decRefBuffer(MemoryBufferOrBuffers buffer); +} \ No newline at end of file diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java new file mode 100644 index 0000000000000000000000000000000000000000..63084b902a0cc684865f18a74621110480138643 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.io.encoded; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A block of data for a given section of a file, similar to VRB but in encoded form. + * Stores a set of buffers for each encoded stream that is a part of each column. + */ +public class EncodedColumnBatch { + /** + * Slice of the data for a stream for some column, stored inside MemoryBuffer's. + * ColumnStreamData can be reused for many EncodedColumnBatch-es (e.g. dictionary stream), so + * it tracks the number of such users via a refcount. + */ + public static class ColumnStreamData { + private List cacheBuffers; + /** Base offset from the beginning of the indexable unit; for example, for ORC, + * offset from the CB in a compressed file, from the stream in uncompressed file. */ + private int indexBaseOffset = 0; + + /** Reference count. */ + private AtomicInteger refCount = new AtomicInteger(0); + + public void reset() { + cacheBuffers.clear(); + refCount.set(0); + indexBaseOffset = 0; + } + + public void incRef() { + refCount.incrementAndGet(); + } + + public int decRef() { + int i = refCount.decrementAndGet(); + assert i >= 0; + return i; + } + + public List getCacheBuffers() { + return cacheBuffers; + } + + public void setCacheBuffers(List cacheBuffers) { + this.cacheBuffers = cacheBuffers; + } + + public int getIndexBaseOffset() { + return indexBaseOffset; + } + + public void setIndexBaseOffset(int indexBaseOffset) { + this.indexBaseOffset = indexBaseOffset; + } + + @Override + public String toString() { + String bufStr = ""; + if (cacheBuffers != null) { + for (MemoryBuffer mb : cacheBuffers) { + bufStr += mb.getClass().getSimpleName() + " with " + mb.getByteBufferRaw().remaining() + " bytes, "; + } + } + return "ColumnStreamData [cacheBuffers=[" + bufStr + + "], indexBaseOffset=" + indexBaseOffset + "]"; + } + + } + + /** The key that is used to map this batch to source location. */ + protected BatchKey batchKey; + /** + * Stream data for each column that has true in the corresponding hasData position. + * For each column, streams are indexed by kind (for ORC), with missing elements being null. + */ + protected ColumnStreamData[][] columnData; + /** Indicates which columns have data. Correspond to columnData elements. */ + protected boolean[] hasData; + + public void reset() { + if (hasData != null) { + Arrays.fill(hasData, false); + } + if (columnData == null) return; + for (int i = 0; i < columnData.length; ++i) { + if (columnData[i] == null) continue; + for (int j = 0; j < columnData[i].length; ++j) { + columnData[i][j] = null; + } + } + } + + + public void initColumn(int colIx, int streamCount) { + hasData[colIx] = true; + if (columnData[colIx] == null || columnData[colIx].length != streamCount) { + columnData[colIx] = new ColumnStreamData[streamCount]; + } + } + + private static final Logger LOG = LoggerFactory.getLogger(EncodedColumnBatch.class); + public void setStreamData(int colIx, int streamIx, ColumnStreamData csd) { + assert hasData[colIx]; + columnData[colIx][streamIx] = csd; + } + + public BatchKey getBatchKey() { + return batchKey; + } + + public ColumnStreamData[] getColumnData(int colIx) { + if (!hasData[colIx]) throw new AssertionError("No data for column " + colIx); + return columnData[colIx]; + } + + public int getTotalColCount() { + return columnData.length; // Includes the columns that have no data + } + + protected void resetColumnArrays(int columnCount) { + if (hasData != null && columnCount == hasData.length) { + Arrays.fill(hasData, false); + return; + } + hasData = new boolean[columnCount]; + ColumnStreamData[][] columnData = new ColumnStreamData[columnCount][]; + if (this.columnData != null) { + for (int i = 0; i < Math.min(columnData.length, this.columnData.length); ++i) { + columnData[i] = this.columnData[i]; + } + } + this.columnData = columnData; + } + + public boolean hasData(int colIx) { + return hasData[colIx]; + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java new file mode 100644 index 0000000000000000000000000000000000000000..201c5e2bc4ee36b1249e687429f22ced9d77c94c --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.io.encoded; + +import java.nio.ByteBuffer; + +/** Abstract interface for any class wrapping a ByteBuffer. */ +public interface MemoryBuffer { + /** + * Get the raw byte buffer that backs this buffer. + * Note - raw buffer should not be modified. + * @return the shared byte buffer + */ + public ByteBuffer getByteBufferRaw(); + public ByteBuffer getByteBufferDup(); +} \ No newline at end of file diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBufferOrBuffers.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBufferOrBuffers.java new file mode 100644 index 0000000000000000000000000000000000000000..c04310a2ba9390a29e86673002e199edff37ba91 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBufferOrBuffers.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.io.encoded; + +public interface MemoryBufferOrBuffers { + MemoryBuffer getSingleBuffer(); + MemoryBuffer[] getMultipleBuffers(); +} \ No newline at end of file diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/DataTypePhysicalVariation.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/DataTypePhysicalVariation.java new file mode 100644 index 0000000000000000000000000000000000000000..778c8c3d793092780b56101ca0054502d7a8f5cc --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/DataTypePhysicalVariation.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +public enum DataTypePhysicalVariation { + NONE, + DECIMAL_64 +} \ No newline at end of file diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java new file mode 100644 index 0000000000000000000000000000000000000000..4484ed27b5a432f96451db0f94a6553074a6dd2e --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java @@ -0,0 +1,759 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.BigInteger; + +/** + * FastHiveDecimal is a mutable fast decimal object. It is the base class for both the + * HiveDecimal and HiveDecimalWritable classes. All fast* methods are protected so they + * cannot be accessed by clients of HiveDecimal and HiveDecimalWritable. HiveDecimal ensures + * it creates new objects when the value changes since it provides immutable semantics; + * HiveDecimalWritable does not create new objects since it provides mutable semantics. + * + * The methods in this class are shells that pickup the member variables from FastHiveDecimal + * parameters and pass them as individual parameters to static methods in the FastHiveDecimalImpl + * class that do the real work. + * + * NOTE: The rationale for fast decimal is in FastHiveDecimalImpl. + */ +public class FastHiveDecimal { + + /* + * We use protected for the fields so the FastHiveDecimalImpl class can access them. Other + * classes including HiveDecimal should not access these fields directly. + */ + + // See FastHiveDecimalImpl for more details on these fields. + + // -1 when negative; 0 when decimal is zero; 1 when positive. + protected int fastSignum; + + // Decimal longwords. + protected long fast2; + protected long fast1; + protected long fast0; + + // The number of integer digits in the decimal. When the integer portion is zero, this is 0. + protected int fastIntegerDigitCount; + + // The scale of the decimal. + protected int fastScale; + + // Used for legacy HiveDecimalV1 setScale compatibility for binary / display serialization of + // trailing zeroes (or rounding). + protected int fastSerializationScale; + + protected FastHiveDecimal() { + fastReset(); + } + + protected FastHiveDecimal(FastHiveDecimal fastDec) { + this(); + fastSignum = fastDec.fastSignum; + fast0 = fastDec.fast0; + fast1 = fastDec.fast1; + fast2 = fastDec.fast2; + fastIntegerDigitCount = fastDec.fastIntegerDigitCount; + fastScale = fastDec.fastScale; + + // Not propagated. + fastSerializationScale = -1; + } + + protected FastHiveDecimal(int fastSignum, FastHiveDecimal fastDec) { + this(); + this.fastSignum = fastSignum; + fast0 = fastDec.fast0; + fast1 = fastDec.fast1; + fast2 = fastDec.fast2; + fastIntegerDigitCount = fastDec.fastIntegerDigitCount; + fastScale = fastDec.fastScale; + + // Not propagated. + fastSerializationScale = -1; + } + + protected FastHiveDecimal( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + this(); + this.fastSignum = fastSignum; + this.fast0 = fast0; + this.fast1 = fast1; + this.fast2 = fast2; + this.fastIntegerDigitCount = fastIntegerDigitCount; + this.fastScale = fastScale; + + fastSerializationScale = -1; + } + + protected FastHiveDecimal(long longValue) { + this(); + FastHiveDecimalImpl.fastSetFromLong(longValue, this); + } + + protected FastHiveDecimal(String string) { + this(); + FastHiveDecimalImpl.fastSetFromString(string, false, this); + } + + protected void fastReset() { + fastSignum = 0; + fast0 = 0; + fast1 = 0; + fast2 = 0; + fastIntegerDigitCount = 0; + fastScale = 0; + fastSerializationScale = -1; + } + + protected void fastSet(FastHiveDecimal fastDec) { + fastSignum = fastDec.fastSignum; + fast0 = fastDec.fast0; + fast1 = fastDec.fast1; + fast2 = fastDec.fast2; + fastIntegerDigitCount = fastDec.fastIntegerDigitCount; + fastScale = fastDec.fastScale; + fastSerializationScale = fastDec.fastSerializationScale; + } + + protected void fastSet( + int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) { + this.fastSignum = fastSignum; + this.fast0 = fast0; + this.fast1 = fast1; + this.fast2 = fast2; + this.fastIntegerDigitCount = fastIntegerDigitCount; + this.fastScale = fastScale; + + // Not specified. + fastSerializationScale = -1; + } + + protected void fastSetSerializationScale(int fastSerializationScale) { + this.fastSerializationScale = fastSerializationScale; + } + + protected int fastSerializationScale() { + return fastSerializationScale; + } + + protected static final String STRING_ENFORCE_PRECISION_OUT_OF_RANGE = + "Decimal precision out of allowed range [1," + HiveDecimal.MAX_PRECISION + "]"; + protected static final String STRING_ENFORCE_SCALE_OUT_OF_RANGE = + "Decimal scale out of allowed range [0," + HiveDecimal.MAX_SCALE + "]"; + protected static final String STRING_ENFORCE_SCALE_LESS_THAN_EQUAL_PRECISION = + "Decimal scale must be less than or equal to precision"; + + protected boolean fastSetFromBigDecimal( + BigDecimal bigDecimal, boolean allowRounding) { + return + FastHiveDecimalImpl.fastSetFromBigDecimal( + bigDecimal, allowRounding, this); + } + + protected boolean fastSetFromBigInteger( + BigInteger bigInteger) { + return + FastHiveDecimalImpl.fastSetFromBigInteger( + bigInteger, this); + } + + protected boolean fastSetFromBigIntegerAndScale( + BigInteger bigInteger, int scale) { + return + FastHiveDecimalImpl.fastSetFromBigInteger( + bigInteger, scale, this); + } + + protected boolean fastSetFromString(String string, boolean trimBlanks) { + byte[] bytes = string.getBytes(); + return + fastSetFromBytes( + bytes, 0, bytes.length, trimBlanks); + } + + protected boolean fastSetFromBytes(byte[] bytes, int offset, int length, boolean trimBlanks) { + return + FastHiveDecimalImpl.fastSetFromBytes( + bytes, offset, length, trimBlanks, this); + } + + protected boolean fastSetFromDigitsOnlyBytesAndScale( + boolean isNegative, byte[] bytes, int offset, int length, int scale) { + return + FastHiveDecimalImpl.fastSetFromDigitsOnlyBytesAndScale( + isNegative, bytes, offset, length, scale, this); + } + + protected void fastSetFromInt(int intValue) { + FastHiveDecimalImpl.fastSetFromInt(intValue, this); + } + + protected void fastSetFromLong(long longValue) { + FastHiveDecimalImpl.fastSetFromLong(longValue, this); + } + + protected boolean fastSetFromLongAndScale(long longValue, int scale) { + return + FastHiveDecimalImpl.fastSetFromLongAndScale( + longValue, scale, this); + } + + protected boolean fastSetFromFloat(float floatValue) { + return + FastHiveDecimalImpl.fastSetFromFloat( + floatValue, this); + } + + protected boolean fastSetFromDouble(double doubleValue) { + return + FastHiveDecimalImpl.fastSetFromDouble( + doubleValue, this); + } + + protected void fastFractionPortion() { + FastHiveDecimalImpl.fastFractionPortion( + fastSignum, fast0, fast1, fast2, fastScale, + this); + } + + protected void fastIntegerPortion() { + FastHiveDecimalImpl.fastIntegerPortion( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale, + this); + } + + protected static final int FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ = 8 * 3; + + protected boolean fastSerializationUtilsRead( + InputStream inputStream, int scale, + byte[] scratchBytes) throws IOException, EOFException { + return + FastHiveDecimalImpl.fastSerializationUtilsRead( + inputStream, scale, scratchBytes, this); + } + + protected boolean fastSetFromBigIntegerBytesAndScale( + byte[] bytes, int offset, int length, int scale) { + return FastHiveDecimalImpl.fastSetFromBigIntegerBytesAndScale( + bytes, offset, length, scale, this); + } + + protected static final int SCRATCH_LONGS_LEN_FAST_SERIALIZATION_UTILS_WRITE = 6; + + protected boolean fastSerializationUtilsWrite(OutputStream outputStream, + long[] scratchLongs) + throws IOException { + return + FastHiveDecimalImpl.fastSerializationUtilsWrite( + outputStream, + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale, + scratchLongs); + } + + /* + * Deserializes 64-bit decimals up to the maximum 64-bit precision (18 decimal digits). + */ + protected void fastDeserialize64(long decimalLong, int scale) { + FastHiveDecimalImpl.fastDeserialize64( + decimalLong, scale, this); + } + + /* + * Serializes decimal64 up to the maximum 64-bit precision (18 decimal digits). + */ + protected long fastSerialize64(int scale) { + return + FastHiveDecimalImpl.fastSerialize64( + scale, + fastSignum, fast1, fast0, fastScale); + } + + // The fastBigIntegerBytes method returns 3 56 bit (7 byte) words and a possible sign byte. + // However, the fastBigIntegerBytes can take on trailing zeroes -- so make it larger. + protected static final int FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES = 1 + 48; + protected static final int FAST_SCRATCH_LONGS_LEN = 6; + + protected int fastBigIntegerBytes( + long[] scratchLongs, byte[] buffer) { + return + FastHiveDecimalImpl.fastBigIntegerBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + fastSerializationScale, + scratchLongs, buffer); + } + + protected int fastBigIntegerBytesScaled( + int serializationScale, + long[] scratchLongs, byte[] buffer) { + return + FastHiveDecimalImpl.fastBigIntegerBytesScaled( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + serializationScale, + scratchLongs, buffer); + } + + protected boolean fastIsByte() { + return + FastHiveDecimalImpl.fastIsByte( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected byte fastByteValueClip() { + return + FastHiveDecimalImpl.fastByteValueClip( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected boolean fastIsShort() { + return + FastHiveDecimalImpl.fastIsShort( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected short fastShortValueClip() { + return + FastHiveDecimalImpl.fastShortValueClip( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected boolean fastIsInt() { + return + FastHiveDecimalImpl.fastIsInt( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected int fastIntValueClip() { + return + FastHiveDecimalImpl.fastIntValueClip( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected boolean fastIsLong() { + return + FastHiveDecimalImpl.fastIsLong( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected long fastLongValueClip() { + return + FastHiveDecimalImpl.fastLongValueClip( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected float fastFloatValue() { + return + FastHiveDecimalImpl.fastFloatValue( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected double fastDoubleValue() { + return + FastHiveDecimalImpl.fastDoubleValue( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected BigInteger fastBigIntegerValue() { + return + FastHiveDecimalImpl.fastBigIntegerValue( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + fastSerializationScale); + } + + protected BigDecimal fastBigDecimalValue() { + return + FastHiveDecimalImpl.fastBigDecimalValue( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale); + } + + protected int fastScale() { + return fastScale; + } + + protected int fastSignum() { + return fastSignum; + } + + protected int fastCompareTo(FastHiveDecimal right) { + return + FastHiveDecimalImpl.fastCompareTo( + fastSignum, fast0, fast1, fast2, + fastScale, + right.fastSignum, right.fast0, right.fast1, right.fast2, + right.fastScale); + } + + protected static int fastCompareTo(FastHiveDecimal left, FastHiveDecimal right) { + return + FastHiveDecimalImpl.fastCompareTo( + left.fastSignum, left.fast0, left.fast1, left.fast2, + left.fastScale, + right.fastSignum, right.fast0, right.fast1, right.fast2, + right.fastScale); + } + + protected boolean fastEquals(FastHiveDecimal that) { + return + FastHiveDecimalImpl.fastEquals( + fastSignum, fast0, fast1, fast2, + fastScale, + that.fastSignum, that.fast0, that.fast1, that.fast2, + that.fastScale); + } + + protected void fastAbs() { + fastSignum = 1; + } + + protected void fastNegate() { + if (fastSignum == 0) { + return; + } + fastSignum = (fastSignum == 1 ? -1 : 1); + } + + protected int fastNewFasterHashCode() { + return + FastHiveDecimalImpl.fastNewFasterHashCode( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected int fastHashCode() { + return + FastHiveDecimalImpl.fastHashCode( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected int fastIntegerDigitCount() { + return fastIntegerDigitCount; + } + + protected int fastSqlPrecision() { + return + FastHiveDecimalImpl.fastSqlPrecision( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale); + } + + protected int fastRawPrecision() { + return + FastHiveDecimalImpl.fastRawPrecision( + fastSignum, fast0, fast1, fast2); + } + + protected boolean fastScaleByPowerOfTen( + int n, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastScaleByPowerOfTen( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale, + n, + fastResult); + } + + protected static String fastRoundingModeToString(int roundingMode) { + String roundingModeString; + switch (roundingMode) { + case BigDecimal.ROUND_DOWN: + roundingModeString = "ROUND_DOWN"; + break; + case BigDecimal.ROUND_UP: + roundingModeString = "ROUND_UP"; + break; + case BigDecimal.ROUND_FLOOR: + roundingModeString = "ROUND_FLOOR"; + break; + case BigDecimal.ROUND_CEILING: + roundingModeString = "ROUND_CEILING"; + break; + case BigDecimal.ROUND_HALF_UP: + roundingModeString = "ROUND_HALF_UP"; + break; + case BigDecimal.ROUND_HALF_EVEN: + roundingModeString = "ROUND_HALF_EVEN"; + break; + default: + roundingModeString = "Unknown"; + } + return roundingModeString + " (" + roundingMode + ")"; + } + + protected boolean fastRound( + int newScale, int roundingMode, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastRound( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + newScale, roundingMode, + fastResult); + } + + protected boolean isAllZeroesBelow( + int power) { + return + FastHiveDecimalImpl.isAllZeroesBelow( + fastSignum, fast0, fast1, fast2, power); + } + + protected boolean fastEnforcePrecisionScale( + int maxPrecision, int maxScale) { + if (maxPrecision <= 0 || maxPrecision > HiveDecimal.MAX_PRECISION) { + return false; + } + if (maxScale < 0 || maxScale > HiveDecimal.MAX_SCALE) { + return false; + } + /* + if (!fastIsValid()) { + fastRaiseInvalidException(); + } + */ + FastCheckPrecisionScaleStatus status = + FastHiveDecimalImpl.fastCheckPrecisionScale( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + maxPrecision, maxScale); + switch (status) { + case NO_CHANGE: + return true; + case OVERFLOW: + return false; + case UPDATE_SCALE_DOWN: + { + if (!FastHiveDecimalImpl.fastUpdatePrecisionScale( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + maxPrecision, maxScale, status, + this)) { + return false; + } + /* + if (!fastIsValid()) { + fastRaiseInvalidException(); + } + */ + return true; + } + default: + throw new RuntimeException("Unknown fast decimal check precision and scale status " + status); + } + } + + protected FastCheckPrecisionScaleStatus fastCheckPrecisionScale( + int maxPrecision, int maxScale) { + return + FastHiveDecimalImpl.fastCheckPrecisionScale( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + maxPrecision, maxScale); + } + + protected static enum FastCheckPrecisionScaleStatus { + NO_CHANGE, + OVERFLOW, + UPDATE_SCALE_DOWN; + } + + protected boolean fastUpdatePrecisionScale( + int maxPrecision, int maxScale, FastCheckPrecisionScaleStatus status, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastUpdatePrecisionScale( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + maxPrecision, maxScale, status, + fastResult); + } + + protected boolean fastAdd( + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastAdd( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + protected boolean fastSubtract( + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastSubtract( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + protected boolean fastMultiply( + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastMultiply( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + protected boolean fastRemainder( + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastRemainder( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + protected boolean fastDivide( + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastDivide( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + protected boolean fastPow( + int exponent, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastPow( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale, + exponent, + fastResult); + } + + protected String fastToString( + byte[] scratchBuffer) { + return + FastHiveDecimalImpl.fastToString( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, -1, + scratchBuffer); + } + + protected String fastToString() { + return + FastHiveDecimalImpl.fastToString( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, -1); + } + + protected String fastToFormatString(int formatScale) { + return + FastHiveDecimalImpl.fastToFormatString( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + formatScale); + } + + protected String fastToFormatString( + int formatScale, + byte[] scratchBuffer) { + return + FastHiveDecimalImpl.fastToFormatString( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + formatScale, + scratchBuffer); + } + + protected String fastToDigitsOnlyString() { + return + FastHiveDecimalImpl.fastToDigitsOnlyString( + fast0, fast1, fast2, + fastIntegerDigitCount); + } + + // Sign, zero, dot, 2 * digits (to support toFormatString which can add a lot of trailing zeroes). + protected final static int FAST_SCRATCH_BUFFER_LEN_TO_BYTES = + 1 + 1 + 1 + 2 * FastHiveDecimalImpl.MAX_DECIMAL_DIGITS; + + protected int fastToBytes( + byte[] scratchBuffer) { + return + FastHiveDecimalImpl.fastToBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, -1, + scratchBuffer); + } + + protected int fastToFormatBytes( + int formatScale, + byte[] scratchBuffer) { + return + FastHiveDecimalImpl.fastToFormatBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + formatScale, + scratchBuffer); + } + + protected int fastToDigitsOnlyBytes( + byte[] scratchBuffer) { + return + FastHiveDecimalImpl.fastToDigitsOnlyBytes( + fast0, fast1, fast2, + fastIntegerDigitCount, + scratchBuffer); + } + + @Override + public String toString() { + return + FastHiveDecimalImpl.fastToString( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, -1); + } + + protected boolean fastIsValid() { + return FastHiveDecimalImpl.fastIsValid(this); + } + + protected void fastRaiseInvalidException() { + FastHiveDecimalImpl.fastRaiseInvalidException(this); + } + + protected void fastRaiseInvalidException(String parameters) { + FastHiveDecimalImpl.fastRaiseInvalidException(this, parameters); + } +} \ No newline at end of file diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..84ff1c64f5c46adf8a82377ce3e6215c1370135c --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java @@ -0,0 +1,9466 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import java.util.Arrays; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; + +/** + * This class is a companion to the FastHiveDecimal class that separates the essential of code + * out of FastHiveDecimal into static methods in this class so that they can be used directly + * by vectorization to implement decimals by storing the fast0, fast1, and fast2 longs and + * the fastSignum, fastScale, etc ints in the DecimalColumnVector class. + */ +public class FastHiveDecimalImpl extends FastHiveDecimal { + + /** + * Representation of fast decimals. + * + * We use 3 long words to store the 38 digits of fast decimals and and 3 integers for sign, + * integer digit count, and scale. + * + * The lower and middle long words store 16 decimal digits each; the high long word has + * 6 decimal digits; total 38 decimal digits. + * + * We do not try and represent fast decimal value as an unsigned 128 bit binary number in 2 longs. + * There are several important reasons for this. + * + * The effort to represent an unsigned 128 integer in 2 Java signed longs is very difficult, + * error prone, hard to debug, and not worth the effort. + * + * The focus here is on reusing memory (i.e. with HiveDecimalWritable) as often as possible. + * Reusing memory is good for grouping of fast decimal objects and related objects in CPU cache + * lines for fast memory access and eliminating the cost of allocating temporary objects and + * reducing the global cost of garbage collection. + * + * In other words, we are focused on avoiding the poor performance of Java general immutable + * objects. + * + * Reducing memory size or being concerned about the memory size of using 3 longs vs. 2 longs + * for 128 unsigned bits is not the focus here. + * + * Besides focusing on reusing memory, storing a limited number (16) decimal digits in the longs + * rather than compacting the value into all binary bits of 2 longs has a surprising benefit. + * + * One big part of implementing decimals turns out to be manipulating decimal digits. + * + * For example, rounding a decimal involves trimming off lower digits or clearing lower digits. + * Since radix 10 digits cannot be masked with binary masks, we use division and multiplication + * using powers of 10. We can easily manipulate the decimal digits in a long word using simple + * integer multiplication / division without doing emulated 128 binary bit multiplication / + * division (e.g. the defunct Decimal128 class). + * + * For example, say we want to scale (round) down the fraction digits of a decimal. + * + * final long divideFactor = powerOfTenTable[scaleDown]; + * final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; + * + * result0 = + * fast0 / divideFactor + * + ((fast1 % divideFactor) * multiplyFactor); + * result1 = + * fast1 / divideFactor + * + ((fast2 % divideFactor) * multiplyFactor); + * result2 = + * fast2 / divideFactor; + * + * It also turns out to do addition and subtraction of decimals with different scales can involve + * overlap using more than 3 long words. Manipulating extra words is a natural extension of + * the existing techniques. + * + * Why is the decimal digits representation easier to debug? You can see the decimal digits in + * the 3 long words and do not have to convert binary words to decimal to see the value. + * + * 16 decimal digits for a long was choose so that an int can have 1/2 or 8 decimal digits during + * multiplication of int half words so intermediate multiplication results do not overflow a long. + * And, so addition overflow is well below the sign bit of a long. + */ + + // Code Sections: + // Initialize (fastSetFrom*). + // Take Integer or Fractional Portion. + // Binary to Decimal Conversion. + // Decimal to Binary Conversion.r + // Emulate SerializationUtils Deserialization used by ORC. + // Emulate SerializationUtils Serialization used by ORC. + // Emulate BigInteger Deserialization used by LazyBinary and others. + // Emulate BigInteger Serialization used by LazyBinary and others. + // Decimal to Integer Conversion. + // Decimal to Non-Integer Conversion. + // Decimal Comparison. + // Decimal Rounding. + // Decimal Scale Up/Down. + // Decimal Precision / Trailing Zeroes. + // Decimal Addition / Subtraction. + // Decimal Multiply. + // Decimal Division / Remainder. + // Decimal String Formatting. + // Decimal Validation. + // Decimal Debugging. + + private static final long[] powerOfTenTable = { + 1L, // 0 + 10L, + 100L, + 1000L, + 10000L, + 100000L, + 1000000L, + 10000000L, + 100000000L, // 8 + 1000000000L, + 10000000000L, + 100000000000L, + 1000000000000L, + 10000000000000L, + 100000000000000L, + 1000000000000000L, + 10000000000000000L, // 16 + 100000000000000000L, + 1000000000000000000L, // 18 + }; + + public static final int MAX_DECIMAL_DIGITS = 38; + + /** + * Int: 8 decimal digits. An even number and 1/2 of MAX_LONGWORD_DECIMAL. + */ + private static final int INTWORD_DECIMAL_DIGITS = 8; + private static final int MULTIPLER_INTWORD_DECIMAL = (int) powerOfTenTable[INTWORD_DECIMAL_DIGITS]; + + /** + * Long: 16 decimal digits. An even number and twice MAX_INTWORD_DECIMAL. + */ + private static final int LONGWORD_DECIMAL_DIGITS = 16; + private static final long MAX_LONGWORD_DECIMAL = powerOfTenTable[LONGWORD_DECIMAL_DIGITS] - 1; + private static final long MULTIPLER_LONGWORD_DECIMAL = powerOfTenTable[LONGWORD_DECIMAL_DIGITS]; + + public static final int DECIMAL64_DECIMAL_DIGITS = 18; + public static final long MAX_ABS_DECIMAL64 = 999999999999999999L; // 18 9's -- quite reliable! + + private static final int TWO_X_LONGWORD_DECIMAL_DIGITS = 2 * LONGWORD_DECIMAL_DIGITS; + private static final int THREE_X_LONGWORD_DECIMAL_DIGITS = 3 * LONGWORD_DECIMAL_DIGITS; + private static final int FOUR_X_LONGWORD_DECIMAL_DIGITS = 4 * LONGWORD_DECIMAL_DIGITS; + + // 38 decimal maximum - 32 digits in 2 lower longs (6 digits here). + private static final int HIGHWORD_DECIMAL_DIGITS = MAX_DECIMAL_DIGITS - TWO_X_LONGWORD_DECIMAL_DIGITS; + private static final long MAX_HIGHWORD_DECIMAL = + powerOfTenTable[HIGHWORD_DECIMAL_DIGITS] - 1; + + // 38 * 2 or 76 full decimal maximum - (64 + 8) digits in 4 lower longs (4 digits here). + private static final long FULL_MAX_HIGHWORD_DECIMAL = + powerOfTenTable[MAX_DECIMAL_DIGITS * 2 - (FOUR_X_LONGWORD_DECIMAL_DIGITS + INTWORD_DECIMAL_DIGITS)] - 1; + + /** + * BigInteger constants. + */ + + private static final BigInteger BIG_INTEGER_TWO = BigInteger.valueOf(2); + private static final BigInteger BIG_INTEGER_FIVE = BigInteger.valueOf(5); + private static final BigInteger BIG_INTEGER_TEN = BigInteger.valueOf(10); + + public static final BigInteger BIG_INTEGER_MAX_DECIMAL = + BIG_INTEGER_TEN.pow(MAX_DECIMAL_DIGITS).subtract(BigInteger.ONE); + + private static final BigInteger BIG_INTEGER_MAX_LONGWORD_DECIMAL = + BigInteger.valueOf(MAX_LONGWORD_DECIMAL); + + private static final BigInteger BIG_INTEGER_LONGWORD_MULTIPLIER = + BigInteger.ONE.add(BIG_INTEGER_MAX_LONGWORD_DECIMAL); + private static final BigInteger BIG_INTEGER_LONGWORD_MULTIPLIER_2X = + BIG_INTEGER_LONGWORD_MULTIPLIER.multiply(BIG_INTEGER_LONGWORD_MULTIPLIER); + private static final BigInteger BIG_INTEGER_MAX_HIGHWORD_DECIMAL = + BigInteger.valueOf(MAX_HIGHWORD_DECIMAL); + private static final BigInteger BIG_INTEGER_HIGHWORD_MULTIPLIER = + BigInteger.ONE.add(BIG_INTEGER_MAX_HIGHWORD_DECIMAL); + + // UTF-8 byte constants used by string/UTF-8 bytes to decimal and decimal to String/UTF-8 byte + // conversion. + + // There is only one blank in UTF-8. + private static final byte BYTE_BLANK = (byte) ' '; + + private static final byte BYTE_DIGIT_ZERO = (byte) '0'; + private static final byte BYTE_DIGIT_NINE = (byte) '9'; + + // Decimal point. + private static final byte BYTE_DOT = (byte) '.'; + + // Sign. + private static final byte BYTE_MINUS = (byte) '-'; + private static final byte BYTE_PLUS = (byte) '+'; + + // Exponent E or e. + private static final byte BYTE_EXPONENT_LOWER = (byte) 'e'; + private static final byte BYTE_EXPONENT_UPPER = (byte) 'E'; + + //************************************************************************************************ + // Initialize (fastSetFrom*). + + /* + * All of the fastSetFrom* methods require the caller to pass a fastResult parameter has been + * reset for better performance. + */ + + private static void doRaiseSetFromBytesInvalid( + byte[] bytes, int offset, int length, + FastHiveDecimal fastResult) { + final int end = offset + length; + throw new RuntimeException( + "Invalid fast decimal \"" + + new String(bytes, offset, end) + "\"" + + " fastSignum " + fastResult.fastSignum + " fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2 + + " fastIntegerDigitCount " + fastResult.fastIntegerDigitCount +" fastScale " + fastResult.fastScale + + " stack trace: " + getStackTraceAsSingleLine(Thread.currentThread().getStackTrace())); + } + + /** + * Scan a byte array slice for a decimal number in UTF-8 bytes. + * + * Syntax: + * [+|-][integerPortion][.[fractionalDigits]][{E|e}[+|-]exponent] + * // Where at least one integer or fractional + * // digit is required... + * + * We handle too many fractional digits by doing rounding ROUND_HALF_UP. + * + * NOTE: The fastSetFromBytes method requires the caller to pass a fastResult parameter has been + * reset for better performance. + * + * @param bytes the bytes to copy from + * @param offset the starting location in bytes + * @param length the number of bytes to use from bytes + * @param trimBlanks should spaces be trimmed? + * @param fastResult True if the byte array slice was successfully converted to a decimal. + * @return Was a valid number found? + */ + public static boolean fastSetFromBytes(byte[] bytes, int offset, int length, boolean trimBlanks, + FastHiveDecimal fastResult) { + + final int bytesLength = bytes.length; + + if (offset < 0 || offset >= bytesLength) { + return false; + } + final int end = offset + length; + if (end <= offset || end > bytesLength) { + return false; + } + + // We start here with at least one byte. + int index = offset; + + if (trimBlanks) { + while (bytes[index] == BYTE_BLANK) { + if (++index >= end) { + return false; + } + } + } + + // Started with a few ideas from BigDecimal(char[] in, int offset, int len) constructor... + // But soon became very fast decimal specific. + + boolean isNegative = false; + if (bytes[index] == BYTE_MINUS) { + isNegative = true; + if (++index >= end) { + return false; + } + } else if (bytes[index] == BYTE_PLUS) { + if (++index >= end) { + return false; + } + } + + int precision = 0; + + // We fill starting with highest digit in highest longword (HIGHWORD_DECIMAL_DIGITS) and + // move down. At end will will shift everything down if necessary. + + int longWordIndex = 0; // Where 0 is the highest longword; 1 is middle longword, etc. + + int digitNum = HIGHWORD_DECIMAL_DIGITS; + long multiplier = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1]; + + int digitValue = 0; + long longWord = 0; + + long fast0 = 0; + long fast1 = 0; + long fast2 = 0; + + byte work; + + // Parse integer portion. + + boolean haveInteger = false; + while (true) { + work = bytes[index]; + if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) { + break; + } + haveInteger = true; + if (precision == 0 && work == BYTE_DIGIT_ZERO) { + // Ignore leading zeroes. + if (++index >= end) { + break; + } + continue; + } + digitValue = work - BYTE_DIGIT_ZERO; + if (digitNum == 0) { + + // Integer parsing move to next lower longword. + + // Save previous longword. + if (longWordIndex == 0) { + fast2 = longWord; + } else if (longWordIndex == 1) { + fast1 = longWord; + } else if (longWordIndex == 2) { + + // We have filled HiveDecimal.MAX_PRECISION digits and have no more room in our limit precision + // fast decimal. + return false; + } + longWordIndex++; + + // The middle and lowest longwords highest digit number is LONGWORD_DECIMAL_DIGITS. + digitNum = LONGWORD_DECIMAL_DIGITS; + multiplier = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - 1]; + longWord = 0; + } + longWord += digitValue * multiplier; + multiplier /= 10; + digitNum--; + precision++; + if (++index >= end) { + break; + } + } + + // At this point we may have parsed an integer. + + // Try to eat a dot now since it could be the end. We remember if we saw a dot so we can + // do error checking later and detect just a dot. + boolean sawDot = false; + if (index < end && bytes[index] == BYTE_DOT) { + sawDot = true; + index++; + } + + // Try to eat trailing blank padding. + if (trimBlanks && index < end && bytes[index] == BYTE_BLANK) { + index++; + while (index < end && bytes[index] == BYTE_BLANK) { + index++; + } + if (index < end) { + // Junk after trailing blank padding. + return false; + } + // Otherwise, fall through and process the what we saw before possible trailing blanks. + } + + // Any more input? + if (index >= end) { + + // We hit the end after getting optional integer and optional dot and optional blank padding. + + if (!haveInteger) { + return false; + } + + if (precision == 0) { + + // We just had leading zeroes (and possibly a dot and trailing blanks). + // Value is 0. + return true; + } + // Save last longword. + if (longWordIndex == 0) { + fast2 = longWord; + } else if (longWordIndex == 1) { + fast1 = longWord; + } else { + fast0 = longWord; + } + fastResult.fastSignum = (isNegative ? -1 : 1); + fastResult.fastIntegerDigitCount = precision; + fastResult.fastScale = 0; + final int scaleDown = HiveDecimal.MAX_PRECISION - precision; + if (scaleDown > 0) { + doFastScaleDown(fast0, fast1, fast2, scaleDown, fastResult); + } else { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } + return true; + } + + // We have more input but did we start with something valid? + if (!haveInteger && !sawDot) { + + // Must have one of those at this point. + return false; + } + + int integerDigitCount = precision; + + int nonTrailingZeroScale = 0; + boolean roundingNecessary = false; + if (sawDot) { + + // Parse fraction portion. + + while (true) { + work = bytes[index]; + if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) { + if (!haveInteger) { + + // Naked dot. + return false; + } + break; + } + digitValue = work - BYTE_DIGIT_ZERO; + if (digitNum == 0) { + + // Fraction digit parsing move to next lower longword. + + // Save previous longword. + if (longWordIndex == 0) { + fast2 = longWord; + } else if (longWordIndex == 1) { + fast1 = longWord; + } else if (longWordIndex == 2) { + + // We have filled HiveDecimal.MAX_PRECISION digits and have no more room in our limit precision + // fast decimal. However, since we are processing fractional digits, we do rounding. + // away. + if (digitValue >= 5) { + roundingNecessary = true; + } + + // Scan through any remaining digits... + while (++index < end) { + work = bytes[index]; + if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) { + break; + } + } + break; + } + longWordIndex++; + digitNum = LONGWORD_DECIMAL_DIGITS; + multiplier = powerOfTenTable[digitNum - 1]; + longWord = 0; + } + longWord += digitValue * multiplier; + multiplier /= 10; + digitNum--; + precision++; + if (digitValue != 0) { + nonTrailingZeroScale = precision - integerDigitCount; + } + if (++index >= end) { + break; + } + } + } + + boolean haveExponent = false; + if (index < end && + (bytes[index] == BYTE_EXPONENT_UPPER || bytes[index] == BYTE_EXPONENT_LOWER)) { + haveExponent = true; + index++; + if (index >= end) { + // More required. + return false; + } + } + + // At this point we have a number. Save it in fastResult. Round it. If we have an exponent, + // we will do a power 10 operation on fastResult. + + // Save last longword. + if (longWordIndex == 0) { + fast2 = longWord; + } else if (longWordIndex == 1) { + fast1 = longWord; + } else { + fast0 = longWord; + } + + int trailingZeroesScale = precision - integerDigitCount; + if (integerDigitCount == 0 && nonTrailingZeroScale == 0) { + // Zero(es). + } else { + fastResult.fastSignum = (isNegative ? -1 : 1); + fastResult.fastIntegerDigitCount = integerDigitCount; + fastResult.fastScale = nonTrailingZeroScale; + final int trailingZeroCount = trailingZeroesScale - fastResult.fastScale; + final int scaleDown = HiveDecimal.MAX_PRECISION - precision + trailingZeroCount; + if (scaleDown > 0) { + doFastScaleDown(fast0, fast1, fast2, scaleDown, fastResult); + } else { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } + } + + if (roundingNecessary) { + + if (fastResult.fastSignum == 0) { + fastResult.fastSignum = (isNegative ? -1 : 1); + fastResult.fast0 = 1; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = HiveDecimal.MAX_SCALE; + } else { + if (!fastAdd( + fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale, + fastResult.fastSignum, 1, 0, 0, 0, trailingZeroesScale, + fastResult)) { + return false; + } + } + } + + if (!haveExponent) { + + // Try to eat trailing blank padding. + if (trimBlanks && index < end && bytes[index] == BYTE_BLANK) { + index++; + while (index < end && bytes[index] == BYTE_BLANK) { + index++; + } + } + if (index < end) { + // Junk after trailing blank padding. + return false; + } + return true; + } + + // At this point, we have seen the exponent letter E or e and have decimal information as: + // isNegative, precision, integerDigitCount, nonTrailingZeroScale, and + // fast0, fast1, fast2. + // + // After we determine the exponent, we will do appropriate scaling and fill in fastResult. + + boolean isExponentNegative = false; + if (bytes[index] == BYTE_MINUS) { + isExponentNegative = true; + if (++index >= end) { + return false; + } + } else if (bytes[index] == BYTE_PLUS) { + if (++index >= end) { + return false; + } + } + + long exponent = 0; + multiplier = 1; + while (true) { + work = bytes[index]; + if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) { + break; + } + if (multiplier > 10) { + // Power of ten way beyond our precision/scale... + return false; + } + digitValue = work - BYTE_DIGIT_ZERO; + if (digitValue != 0 || exponent != 0) { + exponent = exponent * 10 + digitValue; + multiplier *= 10; + } + if (++index >= end) { + break; + } + } + if (isExponentNegative) { + exponent = -exponent; + } + + // Try to eat trailing blank padding. + if (trimBlanks && index < end && bytes[index] == BYTE_BLANK) { + index++; + while (index < end && bytes[index] == BYTE_BLANK) { + index++; + } + } + if (index < end) { + // Junk after exponent. + return false; + } + + + if (integerDigitCount == 0 && nonTrailingZeroScale == 0) { + // Zero(es). + return true; + } + + if (exponent == 0) { + + // No effect since 10^0 = 1. + + } else { + + // We for these input with exponents, we have at this point an intermediate decimal, + // an exponent power, and a result: + // + // intermediate + // input decimal exponent result + // 701E+1 701 scale 0 +1 7010 scale 0 + // 3E+4 3 scale 0 +4 3 scale 0 + // 3.223E+9 3.223 scale 3 +9 3223000000 scale 0 + // 0.009E+10 0.009 scale 4 +10 90000000 scale 0 + // 0.3221E-2 0.3221 scale 4 -2 0.003221 scale 6 + // 0.00223E-20 0.00223 scale 5 -20 0.0000000000000000000000223 scale 25 + // + + if (!fastScaleByPowerOfTen( + fastResult, + (int) exponent, + fastResult)) { + return false; + } + } + + final int trailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (trailingZeroCount > 0) { + doFastScaleDown( + fastResult, + trailingZeroCount, + fastResult); + fastResult.fastScale -= trailingZeroCount; + } + + return true; + } + + /** + * Scans a byte array slice for UNSIGNED RAW DIGITS ONLY in UTF-8 (ASCII) characters + * and forms a decimal from the digits and a sign and scale. + * + * Designed for BinarySortable serialization format that separates the sign and scale + * from the raw digits. + * + * NOTE: The fastSetFromDigitsOnlyBytesAndScale method requires the caller to pass a fastResult + * parameter has been reset for better performance. + * + * @param isNegative is the number negative + * @param bytes the bytes to read from + * @param offset the position to start at + * @param length the number of bytes to read + * @param scale the scale of the number + * @param fastResult an object it into + * @return True if the sign, digits, and scale were successfully converted to a decimal. + */ + public static boolean fastSetFromDigitsOnlyBytesAndScale( + boolean isNegative, byte[] bytes, int offset, int length, int scale, + FastHiveDecimal fastResult) { + + final int bytesLength = bytes.length; + + if (offset < 0 || offset >= bytesLength) { + return false; + } + final int end = offset + length; + if (end <= offset || end > bytesLength) { + return false; + } + + // We start here with at least one byte. + int index = offset; + + // A stripped down version of fastSetFromBytes. + + int precision = 0; + + // We fill starting with highest digit in highest longword (HIGHWORD_DECIMAL_DIGITS) and + // move down. At end will will shift everything down if necessary. + + int longWordIndex = 0; // Where 0 is the highest longword; 1 is middle longword, etc. + + int digitNum = HIGHWORD_DECIMAL_DIGITS; + long multiplier = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1]; + + int digitValue; + long longWord = 0; + + long fast0 = 0; + long fast1 = 0; + long fast2 = 0; + + byte work; + + // Parse digits. + + boolean haveInteger = false; + while (true) { + work = bytes[index]; + if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) { + if (!haveInteger) { + return false; + } + break; + } + haveInteger = true; + if (precision == 0 && work == BYTE_DIGIT_ZERO) { + // Ignore leading zeroes. + if (++index >= end) { + break; + } + continue; + } + digitValue = work - BYTE_DIGIT_ZERO; + if (digitNum == 0) { + + // Integer parsing move to next lower longword. + + // Save previous longword. + if (longWordIndex == 0) { + fast2 = longWord; + } else if (longWordIndex == 1) { + fast1 = longWord; + } else if (longWordIndex == 2) { + + // We have filled HiveDecimal.MAX_PRECISION digits and have no more room in our limit precision + // fast decimal. + return false; + } + longWordIndex++; + + // The middle and lowest longwords highest digit number is LONGWORD_DECIMAL_DIGITS. + digitNum = LONGWORD_DECIMAL_DIGITS; + multiplier = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - 1]; + longWord = 0; + } + longWord += digitValue * multiplier; + multiplier /= 10; + digitNum--; + precision++; + if (++index >= end) { + break; + } + } + + // Just an digits? + if (index < end) { + return false; + } + + if (precision == 0) { + // We just had leading zeroes. + // Value is 0. + return true; + } + + // Save last longword. + if (longWordIndex == 0) { + fast2 = longWord; + } else if (longWordIndex == 1) { + fast1 = longWord; + } else { + fast0 = longWord; + } + fastResult.fastSignum = (isNegative ? -1 : 1); + fastResult.fastIntegerDigitCount = Math.max(0, precision - scale); + fastResult.fastScale = scale; + final int scaleDown = HiveDecimal.MAX_PRECISION - precision; + if (scaleDown > 0) { + doFastScaleDown(fast0, fast1, fast2, scaleDown, fastResult); + } else { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } + return true; + + } + + /** + * Scale down a BigInteger by a power of 10 and round off if necessary using ROUND_HALF_UP. + * @return The scaled and rounded BigInteger. + */ + private static BigInteger doBigIntegerScaleDown(BigInteger unscaledValue, int scaleDown) { + BigInteger[] quotientAndRemainder = unscaledValue.divideAndRemainder(BigInteger.TEN.pow(scaleDown)); + BigInteger quotient = quotientAndRemainder[0]; + BigInteger round = quotientAndRemainder[1].divide(BigInteger.TEN.pow(scaleDown - 1)); + if (round.compareTo(BIG_INTEGER_FIVE) >= 0) { + quotient = quotient.add(BigInteger.ONE); + } + return quotient; + } + + /** + * Create a fast decimal from a BigDecimal. + * + * NOTE: The fastSetFromBigDecimal method requires the caller to pass a fastResult + * parameter has been reset for better performance. + * + * @param bigDecimal the big decimal to copy + * @param allowRounding is rounding allowed? + * @param fastResult an object to reuse + * @return True if the BigDecimal could be converted to our decimal representation. + */ + public static boolean fastSetFromBigDecimal( + BigDecimal bigDecimal, boolean allowRounding, FastHiveDecimal fastResult) { + + // We trim the trailing zero fraction digits so we don't cause unnecessary precision + // overflow later. + if (bigDecimal.signum() == 0) { + if (bigDecimal.scale() != 0) { + + // For some strange reason BigDecimal 0 can have a scale. We do not support that. + bigDecimal = BigDecimal.ZERO; + } + } + + if (!allowRounding) { + if (bigDecimal.signum() != 0) { + BigDecimal bigDecimalStripped = bigDecimal.stripTrailingZeros(); + int stripTrailingZerosScale = bigDecimalStripped.scale(); + // System.out.println("FAST_SET_FROM_BIG_DECIMAL bigDecimal " + bigDecimal); + // System.out.println("FAST_SET_FROM_BIG_DECIMAL bigDecimalStripped " + bigDecimalStripped); + // System.out.println("FAST_SET_FROM_BIG_DECIMAL stripTrailingZerosScale " + stripTrailingZerosScale); + if (stripTrailingZerosScale < 0) { + + // The trailing zeroes extend into the integer part -- we only want to eliminate the + // fractional zero digits. + + bigDecimal = bigDecimal.setScale(0); + } else { + + // Ok, use result with some or all fractional digits stripped. + + bigDecimal = bigDecimalStripped; + } + } + int scale = bigDecimal.scale(); + if (scale < 0 || scale > HiveDecimal.MAX_SCALE) { + return false; + } + // The digits must fit without rounding. + if (!fastSetFromBigInteger(bigDecimal.unscaledValue(), fastResult)) { + return false; + } + if (fastResult.fastSignum != 0) { + fastResult.fastIntegerDigitCount = Math.max(0, fastResult.fastIntegerDigitCount - scale); + fastResult.fastScale = scale; + } + return true; + } + // This method will scale down and round to fit, if necessary. + if (!fastSetFromBigInteger(bigDecimal.unscaledValue(), bigDecimal.scale(), + bigDecimal.precision(), fastResult)) { + return false; + } + return true; + } + + /** + * Scan a String for a decimal number in UTF-8 characters. + * + * NOTE: The fastSetFromString method requires the caller to pass a fastResult + * parameter has been reset for better performance. + * + * @param string the string to parse + * @param trimBlanks should the blanks be trimmed + * @param result an object to reuse + * @return True if the String was successfully converted to a decimal. + */ + public static boolean fastSetFromString( + String string, boolean trimBlanks, FastHiveDecimal result) { + byte[] bytes = string.getBytes(); + return fastSetFromBytes(bytes, 0, bytes.length, trimBlanks, result); + } + + /** + * Creates a scale 0 fast decimal from an int. + * + * NOTE: The fastSetFromString method requires the caller to pass a fastResult + * parameter has been reset for better performance. + * @param intValue the value to set + * @param fastResult an object to reuse + */ + public static void fastSetFromInt(int intValue, FastHiveDecimal fastResult) { + if (intValue == 0) { + // Zero special case. + return; + } + if (intValue > 0) { + fastResult.fastSignum = 1; + } else { + fastResult.fastSignum = -1; + intValue = Math.abs(intValue); + } + // 10 digit int is all in lowest 16 decimal digit longword. + // Since we are creating with scale 0, no fraction digits to zero trim. + fastResult.fast0 = intValue & 0xFFFFFFFFL; + fastResult.fastIntegerDigitCount = + fastLongWordPrecision(fastResult.fast0); + } + + /** + * Creates a scale 0 fast decimal from a long. + * + * NOTE: The fastSetFromLong method requires the caller to pass a fastResult + * parameter has been reset for better performance. + * + * @param longValue the value to set + * @param fastResult an object to reuse + */ + public static void fastSetFromLong( + long longValue, FastHiveDecimal fastResult) { + if (longValue == 0) { + // Zero special case. + return; + } + // Handle minimum integer case that doesn't have abs(). + if (longValue == Long.MIN_VALUE) { + // Split -9,223,372,036,854,775,808 into 16 digit middle and lowest longwords by hand. + fastResult.fastSignum = -1; + fastResult.fast1 = 922L; + fastResult.fast0 = 3372036854775808L; + fastResult.fastIntegerDigitCount = 19; + } else { + if (longValue > 0) { + fastResult.fastSignum = 1; + } else { + fastResult.fastSignum = -1; + longValue = Math.abs(longValue); + } + // Split into 16 digit middle and lowest longwords remainder / division. + fastResult.fast1 = longValue / MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast0 = longValue % MULTIPLER_LONGWORD_DECIMAL; + if (fastResult.fast1 != 0) { + fastResult.fastIntegerDigitCount = + LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast1); + } else { + fastResult.fastIntegerDigitCount = + fastLongWordPrecision(fastResult.fast0); + } + } + return; + } + + /** + * Creates a fast decimal from a long with a specified scale. + * + * NOTE: The fastSetFromLongAndScale method requires the caller to pass a fastResult + * parameter has been reset for better performance. + * + * @param longValue the value to set as a long + * @param scale the scale to use + * @param fastResult an object to reuse + * @return was the conversion successful? + */ + public static boolean fastSetFromLongAndScale( + long longValue, int scale, FastHiveDecimal fastResult) { + + if (scale < 0 || scale > HiveDecimal.MAX_SCALE) { + return false; + } + + fastSetFromLong(longValue, fastResult); + if (scale == 0) { + return true; + } + + if (!fastScaleByPowerOfTen( + fastResult, + -scale, + fastResult)) { + return false; + } + return true; + } + + /** + * Creates fast decimal from a float. + * + * NOTE: The fastSetFromFloat method requires the caller to pass a fastResult + * parameter has been reset for better performance. + * + * @param floatValue the value to set + * @param fastResult an object to reuse + * @return was the conversion successful? + */ + public static boolean fastSetFromFloat( + float floatValue, FastHiveDecimal fastResult) { + + String floatString = Float.toString(floatValue); + return fastSetFromString(floatString, false, fastResult); + + } + + /** + * Creates fast decimal from a double. + * + * NOTE: The fastSetFromDouble method requires the caller to pass a fastResult + * parameter has been reset for better performance. + * + * @param doubleValue the value to set + * @param fastResult an object to reuse + * @return was the conversion successful? + */ + public static boolean fastSetFromDouble( + double doubleValue, FastHiveDecimal fastResult) { + + String doubleString = Double.toString(doubleValue); + return fastSetFromString(doubleString, false, fastResult); + + } + + /** + * Creates a fast decimal from a BigInteger with scale 0. + * + * For efficiency, we assume that fastResult is fastReset. This method does not set the + * fastScale field. + * + * NOTE: The fastSetFromBigInteger method requires the caller to pass a fastResult + * parameter has been reset for better performance. + * + * @param bigInteger the value to set + * @param fastResult an object to reuse + * @return Return true if the BigInteger value fit within HiveDecimal.MAX_PRECISION. Otherwise, + * false for overflow. + */ + public static boolean fastSetFromBigInteger( + BigInteger bigInteger, FastHiveDecimal fastResult) { + + final int signum = bigInteger.signum(); + if (signum == 0) { + // Zero special case. + return true; + } + fastResult.fastSignum = signum; + if (signum == -1) { + bigInteger = bigInteger.negate(); + } + if (bigInteger.compareTo(BIG_INTEGER_LONGWORD_MULTIPLIER) < 0) { + + // Fits in one longword. + fastResult.fast0 = bigInteger.longValue(); + if (fastResult.fast0 == 0) { + fastResult.fastSignum = 0; + } else { + fastResult.fastIntegerDigitCount = fastLongWordPrecision(fastResult.fast0); + } + return true; + } + BigInteger[] quotientAndRemainder = + bigInteger.divideAndRemainder(BIG_INTEGER_LONGWORD_MULTIPLIER); + fastResult.fast0 = quotientAndRemainder[1].longValue(); + BigInteger quotient = quotientAndRemainder[0]; + if (quotient.compareTo(BIG_INTEGER_LONGWORD_MULTIPLIER) < 0) { + + // Fits in two longwords. + fastResult.fast1 = quotient.longValue(); + if (fastResult.fast0 == 0 && fastResult.fast1 == 0) { + // The special case zero logic at the beginning should have caught this. + throw new RuntimeException("Unexpected"); + } else { + fastResult.fastIntegerDigitCount = + LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast1); + } + return true; + } + + // Uses all 3 decimal longs. + quotientAndRemainder = + quotient.divideAndRemainder(BIG_INTEGER_LONGWORD_MULTIPLIER); + fastResult.fast1 = quotientAndRemainder[1].longValue(); + quotient = quotientAndRemainder[0]; + if (quotient.compareTo(BIG_INTEGER_HIGHWORD_MULTIPLIER) >= 0) { + // Overflow. + return false; + } + fastResult.fast2 = quotient.longValue(); + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + } else { + fastResult.fastIntegerDigitCount = + TWO_X_LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(fastResult.fast2); + } + return true; + } + + /** + * Creates a fast decimal from a BigInteger with a specified scale. + * + * NOTE: The fastSetFromBigInteger method requires the caller to pass a fastResult + * parameter has been reset for better performance. + * + * @param bigInteger the value to set as an integer + * @param scale the scale to use + * @param fastResult an object to reuse + * @return True if the BigInteger and scale were successfully converted to a decimal. + */ + public static boolean fastSetFromBigInteger( + BigInteger bigInteger, int scale, FastHiveDecimal fastResult) { + // Poor performance, because the precision will be calculated by bigInteger.toString() + return fastSetFromBigInteger(bigInteger, scale, -1, fastResult); + } + + /** + * Creates a fast decimal from a BigInteger with a specified scale. + * + * NOTE: The fastSetFromBigInteger method requires the caller to pass a fastResult + * parameter has been reset for better performance. + * + * @param bigInteger the value to set as an integer + * @param scale the scale to use + * @param precision the precision to use + * @param fastResult an object to reuse + * @return True if the BigInteger and scale were successfully converted to a decimal. + */ + public static boolean fastSetFromBigInteger( + BigInteger bigInteger, int scale, int precision, FastHiveDecimal fastResult) { + + if (scale < 0) { + + // Multiply by 10^(-scale) to normalize. We do not use negative scale in our representation. + // + // Example: + // 4.172529E+20 has a negative scale -20 since scale is number of digits below the dot. + // 417252900000000000000 normalized as scale 0. + // + bigInteger = bigInteger.multiply(BIG_INTEGER_TEN.pow(-scale)); + scale = 0; + } + + int signum = bigInteger.signum(); + if (signum == 0) { + // Zero. + return true; + } else if (signum == -1) { + // Normalize to positive. + bigInteger = bigInteger.negate(); + } + + if (precision < 0) { + // A slow way to get the number of decimal digits. + precision = bigInteger.toString().length(); + } + + // System.out.println("FAST_SET_FROM_BIG_INTEGER adjusted bigInteger " + bigInteger + " precision " + precision); + + int integerDigitCount = precision - scale; + // System.out.println("FAST_SET_FROM_BIG_INTEGER integerDigitCount " + integerDigitCount + " scale " + scale); + int maxScale; + if (integerDigitCount >= 0) { + if (integerDigitCount > HiveDecimal.MAX_PRECISION) { + return false; + } + maxScale = HiveDecimal.MAX_SCALE - integerDigitCount; + } else { + maxScale = HiveDecimal.MAX_SCALE; + } + // System.out.println("FAST_SET_FROM_BIG_INTEGER maxScale " + maxScale); + + if (scale > maxScale) { + + // A larger scale is ok -- we will knock off lower digits and round. + + final int trimAwayCount = scale - maxScale; + // System.out.println("FAST_SET_FROM_BIG_INTEGER trimAwayCount " + trimAwayCount); + if (trimAwayCount > 1) { + // First, throw away digits below round digit. + BigInteger bigIntegerThrowAwayBelowRoundDigitDivisor = BIG_INTEGER_TEN.pow(trimAwayCount - 1); + bigInteger = bigInteger.divide(bigIntegerThrowAwayBelowRoundDigitDivisor); + } + // System.out.println("FAST_SET_FROM_BIG_INTEGER with round digit bigInteger " + bigInteger + " length " + bigInteger.toString().length()); + + BigInteger[] quotientAndRemainder = bigInteger.divideAndRemainder(BIG_INTEGER_TEN); + // System.out.println("FAST_SET_FROM_BIG_INTEGER quotientAndRemainder " + Arrays.toString(quotientAndRemainder)); + + BigInteger quotient = quotientAndRemainder[0]; + if (quotientAndRemainder[1].intValue() >= 5) { + if (quotient.equals(BIG_INTEGER_MAX_DECIMAL)) { + + // 38 9's digits. + // System.out.println("FAST_SET_FROM_BIG_INTEGER quotient is BIG_INTEGER_MAX_DECIMAL"); + + if (maxScale == 0) { + // No room above for rounding. + return false; + } + + // System.out.println("FAST_SET_FROM_BIG_INTEGER reached here... scale " + scale + " maxScale " + maxScale); + // Rounding results in 10^N. + bigInteger = BIG_INTEGER_TEN.pow(integerDigitCount); + maxScale = 0; + } else { + + // Round up. + bigInteger = quotient.add(BigInteger.ONE); + } + } else { + + // No rounding. + bigInteger = quotient; + } + scale = maxScale; + } + if (!fastSetFromBigInteger(bigInteger, fastResult)) { + return false; + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + } else { + fastResult.fastSignum = signum; + fastResult.fastIntegerDigitCount = Math.max(0, fastResult.fastIntegerDigitCount - scale); + fastResult.fastScale = scale; + + final int trailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, scale); + if (trailingZeroCount > 0) { + doFastScaleDown( + fastResult, + trailingZeroCount, + fastResult); + fastResult.fastScale -= trailingZeroCount; + } + } + + return true; + } + + //************************************************************************************************ + // Take Integer or Fractional Portion. + + /** + * Creates fast decimal from the fraction portion of a fast decimal. + * + * NOTE: The fastFractionPortion method requires the caller to pass a fastResult + * parameter has been reset for better performance. + * + * @param fastSignum the sign of the number (1, 0, or -1) + * @param fast0 high bits + * @param fast1 second word bits + * @param fast2 third word bits + * @param fastScale the scale + * @param fastResult an object to reuse + */ + public static void fastFractionPortion( + int fastSignum, long fast0, long fast1, long fast2, + int fastScale, + FastHiveDecimal fastResult) { + + if (fastSignum == 0 || fastScale == 0) { + fastResult.fastReset(); + return; + } + + // Clear integer portion; keep fraction. + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + if (fastScale < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + final long clearFactor = powerOfTenTable[fastScale]; + + result0 = fast0 % clearFactor; + result1 = 0; + result2 = 0; + + } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS; + + final long clearFactor = powerOfTenTable[adjustedScaleDown]; + + result0 = fast0; + result1 = fast1 % clearFactor; + result2 = 0; + + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS; + + final long clearFactor = powerOfTenTable[adjustedScaleDown]; + + result0 = fast0; + result1 = fast1; + result2 = fast2 % clearFactor; + + } + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastReset(); + } else { + fastResult.fastSet(fastSignum, result0, result1, result2, /* fastIntegerDigitCount */ 0, fastScale); + } + } + + /** + * Creates fast decimal from the integer portion. + * + * NOTE: The fastFractionPortion method requires the caller to pass a fastResult + * parameter has been reset for better performance. + * + * @param fastSignum the sign of the number (1, 0, or -1) + * @param fast0 high bits + * @param fast1 second word bits + * @param fast2 third word bits + * @param fastIntegerDigitCount the number of integer digits + * @param fastScale the scale + * @param fastResult an object to reuse + */ + public static void fastIntegerPortion( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + FastHiveDecimal fastResult) { + + if (fastSignum == 0) { + fastResult.fastReset(); + return; + } + if (fastScale == 0) { + fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + // Scale down no rounding to clear fraction. + fastResult.fastSignum = fastSignum; + doFastScaleDown( + fast0, fast1, fast2, + fastScale, + fastResult); + fastResult.fastIntegerDigitCount = fastIntegerDigitCount; + fastResult.fastScale = 0; + } + + //************************************************************************************************ + // Binary to Decimal Conversion. + + /** + * Convert 3 binary words of N bits each to a fast decimal (scale 0). + * + * The 3 binary words highWord, middleWord, and lowerWord form a large binary value: + * + * highWord * 2^(M+L) + middleWord * 2^L + lowerWord. + * + * Where L is the number of bits in the lower word; M is the number of bits in the middle word. + * We let L and M be different to support the SerializationUtil serialization where the lower + * word is 62 bits and the remaining words are 63 bits... + * + * @param lowerWord the lower internal representation + * @param middleWord the middle internal representation + * @param highWord the high internal representation + * @param middleWordMultiplier 2^L + * @param highWordMultiplier 2^(M+L) + * @param fastResult an object to reuse + * @return True if the conversion of the 3 binary words to decimal was successful. + */ + public static boolean doBinaryToDecimalConversion( + long lowerWord, long middleWord, long highWord, + FastHiveDecimal middleWordMultiplier, + FastHiveDecimal highWordMultiplier, + FastHiveDecimal fastResult) { + + /* + * Challenge: How to do the math to get this raw binary back to our decimal form. + * + * Briefly, for the middle and upper binary words, convert the middle/upper word into a decimal + * long words and then multiply those by the binary word's power of 2. + * + * And, add the multiply results into the result decimal longwords. + * + */ + long result0 = + lowerWord % MULTIPLER_LONGWORD_DECIMAL; + long result1 = + lowerWord / MULTIPLER_LONGWORD_DECIMAL; + long result2 = 0; + + if (middleWord != 0 || highWord != 0) { + + if (highWord == 0) { + + // Form result from lower and middle words. + + if (!fastMultiply5x5HalfWords( + middleWord % MULTIPLER_LONGWORD_DECIMAL, + middleWord / MULTIPLER_LONGWORD_DECIMAL, + 0, + middleWordMultiplier.fast0, middleWordMultiplier.fast1, middleWordMultiplier.fast2, + fastResult)) { + return false; + } + + final long calc0 = + result0 + + fastResult.fast0; + result0 = + calc0 % MULTIPLER_LONGWORD_DECIMAL; + final long calc1 = + calc0 / MULTIPLER_LONGWORD_DECIMAL + + result1 + + fastResult.fast1; + result1 = + calc1 % MULTIPLER_LONGWORD_DECIMAL; + result2 = + calc1 / MULTIPLER_LONGWORD_DECIMAL + + fastResult.fast2; + + } else if (middleWord == 0) { + + // Form result from lower and high words. + + if (!fastMultiply5x5HalfWords( + highWord % MULTIPLER_LONGWORD_DECIMAL, + highWord / MULTIPLER_LONGWORD_DECIMAL, + 0, + highWordMultiplier.fast0, highWordMultiplier.fast1, highWordMultiplier.fast2, + fastResult)) { + return false; + } + + final long calc0 = + result0 + + fastResult.fast0; + result0 = + calc0 % MULTIPLER_LONGWORD_DECIMAL; + final long calc1 = + calc0 / MULTIPLER_LONGWORD_DECIMAL + + result1 + + fastResult.fast1; + result1 = + calc1 % MULTIPLER_LONGWORD_DECIMAL; + result2 = + calc1 / MULTIPLER_LONGWORD_DECIMAL + + fastResult.fast2; + + } else { + + // Form result from lower, middle, and middle words. + + if (!fastMultiply5x5HalfWords( + middleWord % MULTIPLER_LONGWORD_DECIMAL, + middleWord / MULTIPLER_LONGWORD_DECIMAL, + 0, + middleWordMultiplier.fast0, middleWordMultiplier.fast1, middleWordMultiplier.fast2, + fastResult)) { + return false; + } + + long middleResult0 = fastResult.fast0; + long middleResult1 = fastResult.fast1; + long middleResult2 = fastResult.fast2; + + if (!fastMultiply5x5HalfWords( + highWord % MULTIPLER_LONGWORD_DECIMAL, + highWord / MULTIPLER_LONGWORD_DECIMAL, + 0, + highWordMultiplier.fast0, highWordMultiplier.fast1, highWordMultiplier.fast2, + fastResult)) { + return false; + } + + long calc0 = + result0 + + middleResult0 + + fastResult.fast0; + result0 = + calc0 % MULTIPLER_LONGWORD_DECIMAL; + long calc1 = + calc0 / MULTIPLER_LONGWORD_DECIMAL + + result1 + + middleResult1 + + fastResult.fast1; + result1 = + calc1 % MULTIPLER_LONGWORD_DECIMAL; + result2 = + calc1 / MULTIPLER_LONGWORD_DECIMAL + + middleResult2 + + fastResult.fast2; + } + } + + // Let caller set negative sign if necessary. + if (result2 != 0) { + fastResult.fastIntegerDigitCount = TWO_X_LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(result2); + fastResult.fastSignum = 1; + } else if (result1 != 0) { + fastResult.fastIntegerDigitCount = LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(result1); + fastResult.fastSignum = 1; + } else if (result0 != 0) { + fastResult.fastIntegerDigitCount = fastHighWordPrecision(result0); + fastResult.fastSignum = 1; + } else { + fastResult.fastIntegerDigitCount = 0; + fastResult.fastSignum = 0; + } + + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + + return true; + } + + //************************************************************************************************ + // Decimal to Binary Conversion. + + /** + * A helper method that produces a single binary word remainder from a fast decimal (and + * quotient). + * + * The fast decimal is longwords of 16 digits each and we need binary words of 2^N. Since + * we are in decimal form, we have do work to get to convert to binary form. + * + * We effectively need to produce on big binary value (i.e. greater than 64 bits since + * HiveDecimal needs 128 bits of binary which Java does not provide primitive support for) + * from the decimal long words and get the lower N binary bit remainder. + * + * We could try and do decimal division by 2^N to get the (integer) quotient, multiply the + * quotient by 2^N decimal, and finally do a decimal subtract that from the original decimal. + * The resulting decimal can be used to easily get the binary remainder. + * + * However, currently, we do not have fast decimal division. + * + * The "trick" we do here is to remember from your Algebra in school than multiplication and + * division are inverses of each other. + * + * So instead of doing decimal division by 2^N we multiply by the inverse: 2^-N. + * + * We produce 1 binary word (remainder) and a decimal quotient for the higher portion. + * + * @param dividendFast0 The input decimal that will produce a + * single binary word remainder and decimal quotient. + * @param dividendFast1 second word + * @param dividendFast2 third word + * @param fastInverseConst the fast decimal inverse of 2^N = 2^-N + * @param quotientIntegerWordNum the word in the inverse multiplication result + * to find the quotient integer decimal portion + * @param quotientIntegerDigitNum the digit in the result to find the quotient + * integer decimal portion + * @param fastMultiplierConst The fast decimal multiplier for converting the + * quotient integer to the larger number to + * subtract from the input decimal to get the + * remainder. + * @param scratchLongs where to store the result remainder word (index 3) and + * result quotient decimal longwords (indices 0 .. 2) + * @return True if the results were produced without overflow. + */ + public static boolean doDecimalToBinaryDivisionRemainder( + long dividendFast0, long dividendFast1, long dividendFast2, + FastHiveDecimal fastInverseConst, + int quotientIntegerWordNum, + int quotientIntegerDigitNum, + FastHiveDecimal fastMultiplierConst, + long[] scratchLongs) { + + // Multiply by inverse (2^-N) to do the 2^N division. + if (!fastMultiply5x6HalfWords( + dividendFast0, dividendFast1, dividendFast2, + fastInverseConst.fast0, fastInverseConst.fast1, fastInverseConst.fast2, + scratchLongs)) { + // Overflow. + return false; + } + + final long divideFactor = powerOfTenTable[quotientIntegerDigitNum]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - quotientIntegerDigitNum]; + + // Extract the integer portion to get the quotient. + long quotientFast0 = + scratchLongs[quotientIntegerWordNum] / divideFactor + + ((scratchLongs[quotientIntegerWordNum + 1] % divideFactor) * multiplyFactor); + long quotientFast1 = + scratchLongs[quotientIntegerWordNum + 1] / divideFactor + + ((scratchLongs[quotientIntegerWordNum + 2] % divideFactor) * multiplyFactor); + long quotientFast2 = + scratchLongs[quotientIntegerWordNum + 2] / divideFactor; + + // Multiply the integer quotient back out so we can subtract it from the original to get + // the remainder. + if (!fastMultiply5x6HalfWords( + quotientFast0, quotientFast1, quotientFast2, + fastMultiplierConst.fast0, fastMultiplierConst.fast1, fastMultiplierConst.fast2, + scratchLongs)) { + return false; + } + + long quotientMultiplied0 = scratchLongs[0]; + long quotientMultiplied1 = scratchLongs[1]; + long quotientMultiplied2 = scratchLongs[2]; + + if (!doSubtractSameScaleNoUnderflow( + dividendFast0, dividendFast1, dividendFast2, + quotientMultiplied0, quotientMultiplied1, quotientMultiplied2, + scratchLongs)) { + // Underflow. + return false; + } + + long remainderBinaryWord = + scratchLongs[1] * MULTIPLER_LONGWORD_DECIMAL + + scratchLongs[0]; + + // Pack the output into the scratch longs. + scratchLongs[0] = quotientFast0; + scratchLongs[1] = quotientFast1; + scratchLongs[2] = quotientFast2; + + scratchLongs[3] = remainderBinaryWord; + + return true; + } + + /** + * Convert a fast decimal into 3 binary words of N bits each. + * + * The 3 binary words will form a large binary value that is the unsigned unscaled decimal value: + * + * highWord * 2^(M+L) + middleWord * 2^L + lowerWord. + * + * Where L is the number of bits in the lower word; M is the number of bits in the middle word. + * We let L and M be different to support the SerializationUtil serialization where the lower + * word is 62 bits and the remaining words are 63 bits... + * + * The fast decimal is longwords of 16 digits each and we need binary words of 2^N. Since + * we are in decimal form, we have do work to get to convert to binary form. + * + * See the comments for doDecimalToBinaryDivisionRemainder for details on the parameters. + * + * The lowerWord is produced by calling doDecimalToBinaryDivisionRemainder. The quotient from + * that is passed to doDecimalToBinaryDivisionRemainder to produce the middleWord. The final + * quotient is used to produce the highWord. + * + * @return True if the 3 binary words were produced without overflow. Overflow is not expected. + */ + private static boolean doDecimalToBinaryConversion( + long fast0, long fast1, long fast2, + FastHiveDecimal fastInverseConst, + int quotientIntegerWordNum, + int quotientIntegerDigitNum, + FastHiveDecimal fastMultiplierConst, + long[] scratchLongs) { + + long lowerBinaryWord; + long middleBinaryWord = 0; + long highBinaryWord = 0; + + if (fastCompareTo( + 1, + fast0, fast1, fast2, 0, + 1, + fastMultiplierConst.fast0, fastMultiplierConst.fast1, fastMultiplierConst.fast2, 0) < 0) { + + // Optimize: whole decimal fits in one binary word. + + lowerBinaryWord = + fast1 * MULTIPLER_LONGWORD_DECIMAL + + fast0; + + } else { + + // Do division/remainder to get lower binary word; quotient will either be middle decimal + // or be both high and middle decimal that requires another division/remainder. + + if (!doDecimalToBinaryDivisionRemainder( + fast0, fast1, fast2, + fastInverseConst, + quotientIntegerWordNum, + quotientIntegerDigitNum, + fastMultiplierConst, + scratchLongs)) { + // Overflow. + return false; + } + + // Unpack the output. + long quotientFast0 = scratchLongs[0]; + long quotientFast1 = scratchLongs[1]; + long quotientFast2 = scratchLongs[2]; + + lowerBinaryWord = scratchLongs[3]; + + if (fastCompareTo( + 1, + quotientFast0, quotientFast1, quotientFast2, 0, + 1, + fastMultiplierConst.fast0, fastMultiplierConst.fast1, fastMultiplierConst.fast2, 0) < 0) { + + // Optimize: whole decimal fits in two binary words. + + middleBinaryWord = + quotientFast1 * MULTIPLER_LONGWORD_DECIMAL + + quotientFast0; + + } else { + if (!doDecimalToBinaryDivisionRemainder( + quotientFast0, quotientFast1, quotientFast2, + fastInverseConst, + quotientIntegerWordNum, + quotientIntegerDigitNum, + fastMultiplierConst, + scratchLongs)) { + // Overflow. + return false; + } + + highBinaryWord = + scratchLongs[1] * MULTIPLER_LONGWORD_DECIMAL + + scratchLongs[0]; + + middleBinaryWord = scratchLongs[3]; + + } + } + + scratchLongs[0] = lowerBinaryWord; + scratchLongs[1] = middleBinaryWord; + scratchLongs[2] = highBinaryWord; + + return true; + } + + //************************************************************************************************ + // Emulate SerializationUtils Deserialization used by ORC. + + /* + * fastSerializationUtilsRead lower word is 62 bits (the lower bit is used as the sign and is + * removed). So, we need a multiplier 2^62 + * + * 2^62 = + * 4611686018427387904 or + * 4,611,686,018,427,387,904 or + * 461,1686018427387904 (16 digit comma'd) + */ + private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_62 = + new FastHiveDecimal(1, 1686018427387904L, 461L, 0, 19, 0); + + /* + * fastSerializationUtilsRead middle word is 63 bits. So, we need a multiplier 2^63 + * + * 2^63 = + * 9223372036854775808 (Long.MAX_VALUE) or + * 9,223,372,036,854,775,808 or + * 922,3372036854775808 (16 digit comma'd) + */ + private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_63 = + new FastHiveDecimal(1, 3372036854775808L, 922L, 0, 19, 0); + + /* + * fastSerializationUtilsRead high word multiplier: + * + * Multiply by 2^(62 + 63) -- 38 digits or 3 decimal words. + * + * (2^62)*(2^63) = + * 42535295865117307932921825928971026432 or + * (12345678901234567890123456789012345678) + * ( 1 2 3 ) + * 42,535,295,865,117,307,932,921,825,928,971,026,432 or + * 425352,9586511730793292,1825928971026432 (16 digit comma'd) + */ + private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_125 = + new FastHiveDecimal(1, 1825928971026432L, 9586511730793292L, 425352L, 38, 0); + + /* + * Inverse of 2^63 = 2^-63. Please see comments for doDecimalToBinaryDivisionRemainder. + * + * Multiply by 1/2^63 = 1.08420217248550443400745280086994171142578125e-19 to divide by 2^63. + * As 16 digit comma'd 1084202172485,5044340074528008,6994171142578125 + * + * Scale down: 63 = 44 fraction digits + 19 (negative exponent or number of zeros after dot). + * + * 3*16 (48) + 15 --> 63 down shift. + */ + private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_63_INVERSE = + new FastHiveDecimal(1, 6994171142578125L, 5044340074528008L, 1084202172485L, 45, 0); + + /* + * Where in the inverse multiplication result to find the quotient integer decimal portion. + * + * Please see comments for doDecimalToBinaryDivisionRemainder. + */ + private static final int SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_WORD_NUM = 3; + private static final int SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_DIGIT_NUM = 15; + + /** + * Deserialize data written in the format used by the SerializationUtils methods + * readBigInteger/writeBigInteger and create a decimal using the supplied scale. + * + * ORC uses those SerializationUtils methods for its serialization. + * + * A scratch bytes array is necessary to do the binary to decimal conversion for better + * performance. Pass a FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ byte array for + * scratchBytes. + * + * @param inputStream the stream to read from + * @param scale the scale of the number + * @param scratchBytes An array for the binary to decimal conversion for better + * performance. Must have length of + * FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ. + * @param fastResult an object to reuse + * @return The deserialized decimal or null if the conversion failed. + * @throws IOException failures in reading the stream + */ + public static boolean fastSerializationUtilsRead(InputStream inputStream, int scale, + byte[] scratchBytes, + FastHiveDecimal fastResult) throws IOException { + + // Following a suggestion from Gopal, quickly read in the bytes from the stream. + // CONSIDER: Have ORC read the whole input stream into a big byte array with one call to + // the read(byte[] b, int off, int len) method and then let this method read from the big + // byte array. + int readCount = 0; + int input; + do { + input = inputStream.read(); + if (input == -1) { + throw new EOFException("Reading BigInteger past EOF from " + inputStream); + } + scratchBytes[readCount++] = (byte) input; + } while (input >= 0x80); + + /* + * Determine the 3 binary words like what SerializationUtils.readBigInteger does. + */ + + long lowerWord63 = 0; + long middleWord63 = 0; + long highWord63 = 0; + + long work = 0; + int offset = 0; + int readIndex = 0; + long b; + do { + b = scratchBytes[readIndex++]; + work |= (0x7f & b) << (offset % 63); + offset += 7; + // if we've read 63 bits, roll them into the result + if (offset == 63) { + lowerWord63 = work; + work = 0; + } else if (offset % 63 == 0) { + if (offset == 126) { + middleWord63 = work; + } else if (offset == 189) { + highWord63 = work; + } else { + throw new EOFException("Reading more than 3 words of BigInteger"); + } + work = 0; + } + } while (readIndex < readCount); + + if (work != 0) { + if (offset < 63) { + lowerWord63 = work; + } else if (offset < 126) { + middleWord63 = work; + } else if (offset < 189) { + highWord63 =work; + } else { + throw new EOFException("Reading more than 3 words of BigInteger"); + } + } + + // Grab sign bit and shift it away. + boolean isNegative = ((lowerWord63 & 0x1) != 0); + lowerWord63 >>= 1; + + /* + * Use common binary to decimal conversion method we share with fastSetFromBigIntegerBytes. + */ + if (!doBinaryToDecimalConversion( + lowerWord63, middleWord63, highWord63, + FAST_HIVE_DECIMAL_TWO_POWER_62, + FAST_HIVE_DECIMAL_TWO_POWER_125, // 2^(62 + 63) + fastResult)) { + return false; + } + + if (isNegative) { + + // Adjust negative result, again doing what SerializationUtils.readBigInteger does. + if (!doAddSameScaleSameSign( + /* resultSignum */ 1, + fastResult.fast0, fastResult.fast1, fastResult.fast2, + 1, 0, 0, + fastResult)) { + return false; + } + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + } else { + fastResult.fastSignum = (isNegative ? -1 : 1); + final int rawPrecision = fastRawPrecision(fastResult); + fastResult.fastIntegerDigitCount = Math.max(0, rawPrecision - scale); + fastResult.fastScale = scale; + + /* + * Just in case we deserialize a decimal with trailing zeroes... + */ + final int resultTrailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (resultTrailingZeroCount > 0) { + doFastScaleDown( + fastResult, + resultTrailingZeroCount, + fastResult); + + fastResult.fastScale -= resultTrailingZeroCount; + } + } + + return true; + } + + //************************************************************************************************ + // Emulate SerializationUtils Serialization used by ORC. + + /** + * Write the value of this decimal just like SerializationUtils.writeBigInteger. It header + * comments are: + * + * Write the arbitrarily sized signed BigInteger in vint format. + * + * Signed integers are encoded using the low bit as the sign bit using zigzag + * encoding. + * + * Each byte uses the low 7 bits for data and the high bit for stop/continue. + * + * Bytes are stored LSB first. + * + * NOTE: + * SerializationUtils.writeBigInteger sometimes pads the result with extra zeroes due to + * BigInteger.bitLength -- we do not emulate that. SerializationUtils.readBigInteger will + * produce the same result for both. + * + * @param outputStream the stream to write to + * @param fastSignum the sign digit (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 of the internal representation + * @param fast2 word 2 of the internal representation + * @param fastIntegerDigitCount unused + * @param fastScale unused + * @param scratchLongs scratch space + * @return True if the decimal was successfully serialized into the output stream. + * @throws IOException for problems in writing + */ + public static boolean fastSerializationUtilsWrite(OutputStream outputStream, + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + long[] scratchLongs) + throws IOException { + + boolean isNegative = (fastSignum == -1); + + /* + * The sign is encoded as the least significant bit. + * + * We need to adjust our decimal before conversion to binary. + * + * Positive: + * Multiply by 2. + * + * Negative: + * Logic in SerializationUtils.writeBigInteger does a negate on the BigInteger. We + * do not have to since FastHiveDecimal stores the numbers unsigned in fast0, fast1, + * and fast2. We do need to subtract one though. + * + * And then multiply by 2 and add in the 1 sign bit. + * + * CONSIDER: This could be combined. + */ + long adjust0; + long adjust1; + long adjust2; + + if (isNegative) { + + // Subtract 1. + long r0 = fast0 - 1; + long r1; + if (r0 < 0) { + adjust0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = fast1 - 1; + } else { + adjust0 = r0; + r1 = fast1; + } + if (r1 < 0) { + adjust1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + adjust2 = fast2 - 1; + } else { + adjust1 = r1; + adjust2 = fast2; + } + if (adjust2 < 0) { + return false; + } + + // Now multiply by 2 and add 1 sign bit. + r0 = adjust0 * 2 + 1; + adjust0 = + r0 % MULTIPLER_LONGWORD_DECIMAL; + r1 = + adjust1 * 2 + + r0 / MULTIPLER_LONGWORD_DECIMAL; + adjust1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + adjust2 = + adjust2 * 2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + + } else { + + // Multiply by 2 to make room for 0 sign bit. + long r0 = fast0 * 2; + adjust0 = + r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fast1 * 2 + + r0 / MULTIPLER_LONGWORD_DECIMAL; + adjust1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + adjust2 = + fast2 * 2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + + } + + /* + * Use common decimal to binary conversion method we share with fastBigIntegerBytes. + */ + if (!doDecimalToBinaryConversion( + adjust0, adjust1, adjust2, + FAST_HIVE_DECIMAL_TWO_POWER_63_INVERSE, + SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_WORD_NUM, + SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_DIGIT_NUM, + FAST_HIVE_DECIMAL_TWO_POWER_63, + scratchLongs)) { + // Overflow. + return false; + } + + long lowerWord63 = scratchLongs[0]; + long middleWord63 = scratchLongs[1]; + long highWord63 = scratchLongs[2]; + + int wordCount; + if (highWord63 != 0) { + wordCount = 3; + } else if (middleWord63 != 0) { + wordCount = 2; + } else { + wordCount = 1; + } + + // Write out the first 63 bits worth of data. + long lowBits = lowerWord63; + for(int i=0; i < 9; ++i) { + // If this is the last byte, leave the high bit off + if (wordCount == 1 && (lowBits & ~0x7f) == 0) { + outputStream.write((byte) lowBits); + return true; + } else { + outputStream.write((byte) (0x80 | (lowBits & 0x7f))); + lowBits >>>= 7; + } + } + if (wordCount <= 1) { + throw new RuntimeException("Expecting write word count > 1"); + } + + lowBits = middleWord63; + for(int i=0; i < 9; ++i) { + // If this is the last byte, leave the high bit off + if (wordCount == 2 && (lowBits & ~0x7f) == 0) { + outputStream.write((byte) lowBits); + return true; + } else { + outputStream.write((byte) (0x80 | (lowBits & 0x7f))); + lowBits >>>= 7; + } + } + + lowBits = highWord63; + for(int i=0; i < 9; ++i) { + // If this is the last byte, leave the high bit off + if ((lowBits & ~0x7f) == 0) { + outputStream.write((byte) lowBits); + return true; + } else { + outputStream.write((byte) (0x80 | (lowBits & 0x7f))); + lowBits >>>= 7; + } + } + + // Should not get here. + throw new RuntimeException("Unexpected"); + } + + public static long getDecimal64AbsMax(int precision) { + return powerOfTenTable[precision] - 1; + } + + /* + * Deserializes 64-bit decimals up to the maximum 64-bit precision (18 decimal digits). + * + * NOTE: Major assumption: the input decimal64 has already been bounds checked and a least + * has a precision <= DECIMAL64_DECIMAL_DIGITS. We do not bounds check here for better + * performance. + */ + public static void fastDeserialize64( + final long inputDecimal64Long, final int inputScale, + FastHiveDecimal fastResult) { + + long decimal64Long; + if (inputDecimal64Long == 0) { + fastResult.fastReset(); + return; + } else if (inputDecimal64Long > 0) { + fastResult.fastSignum = 1; + decimal64Long = inputDecimal64Long; + } else { + fastResult.fastSignum = -1; + decimal64Long = -inputDecimal64Long; + } + + // Trim trailing zeroes -- but only below the decimal point. + int trimScale = inputScale; + while (trimScale > 0 && decimal64Long % 10 == 0) { + decimal64Long /= 10; + trimScale--; + } + + fastResult.fast2 = 0; + fastResult.fast1 = decimal64Long / MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast0 = decimal64Long % MULTIPLER_LONGWORD_DECIMAL; + + fastResult.fastScale = trimScale; + + fastResult.fastIntegerDigitCount = + Math.max(0, fastRawPrecision(fastResult) - fastResult.fastScale); + } + + /* + * Serializes decimal64 up to the maximum 64-bit precision (18 decimal digits). + * + * NOTE: Major assumption: the fast decimal has already been bounds checked and a least + * has a precision <= DECIMAL64_DECIMAL_DIGITS. We do not bounds check here for better + * performance. + */ + public static long fastSerialize64( + int scale, + int fastSignum, long fast1, long fast0, int fastScale) { + + if (fastSignum == 0) { + return 0; + } else if (fastSignum == 1) { + return (fast1 * MULTIPLER_LONGWORD_DECIMAL + fast0) * powerOfTenTable[scale - fastScale]; + } else { + return -(fast1 * MULTIPLER_LONGWORD_DECIMAL + fast0) * powerOfTenTable[scale - fastScale]; + } + } + + //************************************************************************************************ + // Emulate BigInteger deserialization used by LazyBinary and others. + + /* + * fastSetFromBigIntegerBytes word size we choose is 56 bits to stay below the 64 bit sign bit: + * So, we need a multiplier 2^56 + * + * 2^56 = + * 72057594037927936 or + * 72,057,594,037,927,936 or + * 7,2057594037927936 (16 digit comma'd) + */ + private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_56 = + new FastHiveDecimal(1, 2057594037927936L, 7L, 0, 17, 0); + + /* + * fastSetFromBigIntegerBytes high word multiplier is 2^(56 + 56) + * + * (2^56)*(2^56) = + * 5192296858534827628530496329220096 or + * (1234567890123456789012345678901234) + * ( 1 2 3 ) + * 5,192,296,858,534,827,628,530,496,329,220,096 or + * 51,9229685853482762,8530496329220096 (16 digit comma'd) + */ + private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_112 = + new FastHiveDecimal(1, 8530496329220096L, 9229685853482762L, 51L, 34, 0); + + // Multiply by 1/2^56 or 1.387778780781445675529539585113525390625e-17 to divide by 2^56. + // As 16 digit comma'd 13877787,8078144567552953,9585113525390625 + // + // Scale down: 56 = 39 fraction digits + 17 (negative exponent or number of zeros after dot). + // + // 3*16 (48) + 8 --> 56 down shift. + // + private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_56_INVERSE = + new FastHiveDecimal(1, 9585113525390625L, 8078144567552953L, 13877787L, 40, 0); + + /* + * Where in the inverse multiplication result to find the quotient integer decimal portion. + * + * Please see comments for doDecimalToBinaryDivisionRemainder. + */ + private static final int BIG_INTEGER_BYTES_QUOTIENT_INTEGER_WORD_NUM = 3; + private static final int BIG_INTEGER_BYTES_QUOTIENT_INTEGER_DIGIT_NUM = 8; + + private static final int INITIAL_SHIFT = 48; // 56 bits minus 1 byte. + + // Long masks and values. + private static final long LONG_56_BIT_MASK = 0xFFFFFFFFFFFFFFL; + private static final long LONG_TWO_TO_56_POWER = LONG_56_BIT_MASK + 1L; + private static final long LONG_BYTE_MASK = 0xFFL; + private static final long LONG_BYTE_HIGH_BIT_MASK = 0x80L; + + // Byte values. + private static final byte BYTE_ALL_BITS = (byte) 0xFF; + + /** + * Convert bytes in the format used by BigInteger's toByteArray format (and accepted by its + * constructor) into a decimal using the specified scale. + * + * Our bigIntegerBytes methods create bytes in this format, too. + * + * This method is designed for high performance and does not create an actual BigInteger during + * binary to decimal conversion. + * + * @param bytes the bytes to read from + * @param offset the starting position in the bytes array + * @param length the number of bytes to read from the bytes array + * @param scale the scale of the number + * @param fastResult an object to reused + * @return did the conversion succeed? + */ + public static boolean fastSetFromBigIntegerBytesAndScale( + byte[] bytes, int offset, int length, int scale, + FastHiveDecimal fastResult) { + + final int bytesLength = bytes.length; + + if (offset < 0 || offset >= bytesLength) { + return false; + } + final int end = offset + length; + if (end <= offset || end > bytesLength) { + return false; + } + + final int startOffset = offset; + + // Roughly based on BigInteger code. + + boolean isNegative = (bytes[offset] < 0); + if (isNegative) { + + // Find first non-sign (0xff) byte of input. + while (offset < end) { + if (bytes[offset] != -1) { + break; + } + offset++; + } + if (offset > end) { + return false; + } + } else { + + // Strip leading zeroes -- although there shouldn't be any for a decimal. + + while (offset < end && bytes[offset] == 0) { + offset++; + } + if (offset >= end) { + // Zero. + return true; + } + } + + long lowerWord56 = 0; + long middleWord56 = 0; + long highWord56 = 0; + + int reverseIndex = end; + + long work; + int shift; + + final int lowestCount = Math.min(reverseIndex - offset, 7); + shift = 0; + for (int i = 0; i < lowestCount; i++) { + work = bytes[--reverseIndex] & 0xFF; + lowerWord56 |= work << shift; + shift += 8; + } + + if (reverseIndex <= offset) { + if (isNegative) { + lowerWord56 = ~lowerWord56 & ((1L << shift) - 1); + } + } else { + + // Go on to middle word. + + final int middleCount = Math.min(reverseIndex - offset, 7); + shift = 0; + for (int i = 0; i < middleCount; i++) { + work = bytes[--reverseIndex] & 0xFF; + middleWord56 |= work << shift; + shift += 8; + } + if (reverseIndex <= offset) { + if (isNegative) { + lowerWord56 = ~lowerWord56 & LONG_56_BIT_MASK; + middleWord56 = ~middleWord56 & ((1L << shift) - 1); + } + } else { + + // Go on to high word. + + final int highCount = Math.min(reverseIndex - offset, 7); + shift = 0; + for (int i = 0; i < highCount; i++) { + work = bytes[--reverseIndex] & 0xFF; + highWord56 |= work << shift; + shift += 8; + } + if (isNegative) { + // We only need to apply negation to all 3 words when there are 3 words, etc. + lowerWord56 = ~lowerWord56 & LONG_56_BIT_MASK; + middleWord56 = ~middleWord56 & LONG_56_BIT_MASK; + highWord56 = ~highWord56 & ((1L << shift) - 1); + } + } + } + + if (!doBinaryToDecimalConversion( + lowerWord56, middleWord56, highWord56, + FAST_HIVE_DECIMAL_TWO_POWER_56, + FAST_HIVE_DECIMAL_TWO_POWER_112, // 2^(56 + 56) + fastResult)) { + // Overflow. Use slower alternate. + return doAlternateSetFromBigIntegerBytesAndScale( + bytes, startOffset, length, scale, + fastResult); + } + + // System.out.println("fastSetFromBigIntegerBytesAndScale fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2); + if (isNegative) { + if (!doAddSameScaleSameSign( + /* resultSignum */ 1, + fastResult.fast0, fastResult.fast1, fastResult.fast2, + 1, 0, 0, + fastResult)) { + // Overflow. Use slower alternate. + return doAlternateSetFromBigIntegerBytesAndScale( + bytes, startOffset, length, scale, + fastResult); + } + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + } else { + fastResult.fastSignum = (isNegative ? -1 : 1); + fastResult.fastScale = scale; + final int rawPrecision = fastRawPrecision(fastResult); + fastResult.fastIntegerDigitCount = Math.max(0, rawPrecision - scale); + + /* + * Just in case we deserialize a decimal with trailing zeroes... + */ + final int resultTrailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (resultTrailingZeroCount > 0) { + doFastScaleDown( + fastResult, + resultTrailingZeroCount, + fastResult); + + fastResult.fastScale -= resultTrailingZeroCount; + } + } + + return true; + } + + /** + * When fastSetFromBigIntegerBytesAndScale can handle the input because it is too large, + * we fall back to this. + */ + private static boolean doAlternateSetFromBigIntegerBytesAndScale( + byte[] bytes, int offset, int length, int scale, + FastHiveDecimal fastResult) { + + byte[] byteArray = Arrays.copyOfRange(bytes, offset, offset + length); + + BigInteger bigInteger = new BigInteger(byteArray); + // System.out.println("doAlternateSetFromBigIntegerBytesAndScale bigInteger " + bigInteger); + BigDecimal bigDecimal = new BigDecimal(bigInteger, scale); + // System.out.println("doAlternateSetFromBigIntegerBytesAndScale bigDecimal " + bigDecimal); + fastResult.fastReset(); + return fastSetFromBigDecimal(bigDecimal, true, fastResult); + } + + //************************************************************************************************ + // Emulate BigInteger serialization used by LazyBinary, Avro, Parquet, and possibly others. + + public static int fastBigIntegerBytes( + final int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int fastSerializeScale, + long[] scratchLongs, byte[] buffer) { + if (fastSerializeScale != -1) { + return + fastBigIntegerBytesScaled( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + fastSerializeScale, + scratchLongs, buffer); + } else { + return + fastBigIntegerBytesUnscaled( + fastSignum, fast0, fast1, fast2, + scratchLongs, buffer); + } + } + + /** + * Return binary representation of this decimal's BigInteger equivalent unscaled value using + * the format that the BigInteger's toByteArray method returns (and the BigInteger constructor + * accepts). + * + * Used by LazyBinary, Avro, and Parquet serialization. + * + * Scratch objects necessary to do the decimal to binary conversion without actually creating a + * BigInteger object are passed for better performance. + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param scratchLongs scratch array of SCRATCH_LONGS_LEN longs + * @param buffer scratch array of SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes + * @return The number of bytes used for the binary result in buffer. Otherwise, 0 if the + * conversion failed. + */ + public static int fastBigIntegerBytesUnscaled( + final int fastSignum, long fast0, long fast1, long fast2, + long[] scratchLongs, byte[] buffer) { + + /* + * Algorithm: + * 1) Convert decimal to three 56-bit words (three is enough for the decimal since we + * represent the decimal with trailing zeroes trimmed). + * 2) Skip leading zeroes in the words. + * 3) Once we find real data (i.e. a non-zero byte), add a sign byte to buffer if necessary. + * 4) Add bytes from the (rest of) 56-bit words. + * 5) Return byte count. + */ + + if (fastSignum == 0) { + buffer[0] = 0; + return 1; + } + + boolean isNegative = (fastSignum == -1); + + /* + * Use common conversion method we share with fastSerializationUtilsWrite. + */ + if (!doDecimalToBinaryConversion( + fast0, fast1, fast2, + FAST_HIVE_DECIMAL_TWO_POWER_56_INVERSE, + BIG_INTEGER_BYTES_QUOTIENT_INTEGER_WORD_NUM, + BIG_INTEGER_BYTES_QUOTIENT_INTEGER_DIGIT_NUM, + FAST_HIVE_DECIMAL_TWO_POWER_56, + scratchLongs)) { + // Overflow. This is not expected. + return 0; + } + + int byteIndex = 0; + + long word0 = scratchLongs[0]; + long word1 = scratchLongs[1]; + long word2 = scratchLongs[2]; + + if (!isNegative) { + + // Positive number. + + long longWork = 0; + + int shift = INITIAL_SHIFT; + + if (word2 != 0L) { + + // Skip leading zeroes in word2. + + while (true) { + longWork = (word2 >> shift) & LONG_BYTE_MASK; + if (longWork != 0) { + break; + } + if (shift == 0) { + throw new RuntimeException("Unexpected #1"); + } + shift -= Byte.SIZE; + } + + // Now that we have found real data, emit sign byte if necessary. + if ((longWork & LONG_BYTE_HIGH_BIT_MASK) != 0) { + // Add sign byte since high bit is on. + buffer[byteIndex++] = (byte) 0; + } + + // Emit the rest of word2 + while (true) { + buffer[byteIndex++] = (byte) longWork; + if (shift == 0) { + break; + } + shift -= Byte.SIZE; + longWork = (word2 >> shift) & LONG_BYTE_MASK; + } + + shift = INITIAL_SHIFT; + } + + if (byteIndex == 0 && word1 == 0L) { + + // Skip word1, also. + + } else { + + if (byteIndex == 0) { + + // Skip leading zeroes in word1. + + while (true) { + longWork = (word1 >> shift) & LONG_BYTE_MASK; + if (longWork != 0) { + break; + } + if (shift == 0) { + throw new RuntimeException("Unexpected #2"); + } + shift -= Byte.SIZE; + } + + // Now that we have found real data, emit sign byte if necessary. + if ((longWork & LONG_BYTE_HIGH_BIT_MASK) != 0) { + // Add sign byte since high bit is on. + buffer[byteIndex++] = (byte) 0; + } + + } else { + longWork = (word1 >> shift) & LONG_BYTE_MASK; + } + + // Emit the rest of word1 + + while (true) { + buffer[byteIndex++] = (byte) longWork; + if (shift == 0) { + break; + } + shift -= Byte.SIZE; + longWork = (word1 >> shift) & LONG_BYTE_MASK; + } + + shift = INITIAL_SHIFT; + } + + if (byteIndex == 0) { + + // Skip leading zeroes in word0. + + while (true) { + longWork = (word0 >> shift) & LONG_BYTE_MASK; + if (longWork != 0) { + break; + } + if (shift == 0) { + + // All zeroes -- we should have handled this earlier. + throw new RuntimeException("Unexpected #3"); + } + shift -= Byte.SIZE; + } + + // Now that we have found real data, emit sign byte if necessary. + if ((longWork & LONG_BYTE_HIGH_BIT_MASK) != 0) { + // Add sign byte since high bit is on. + buffer[byteIndex++] = (byte) 0; + } + + } else { + longWork = (word0 >> shift) & LONG_BYTE_MASK; + } + + // Emit the rest of word0. + while (true) { + buffer[byteIndex++] = (byte) longWork; + if (shift == 0) { + break; + } + shift -= Byte.SIZE; + longWork = (word0 >> shift) & LONG_BYTE_MASK; + } + + } else { + + // Negative number. + + // Subtract 1 for two's compliment adjustment. + word0--; + if (word0 < 0) { + word0 += LONG_TWO_TO_56_POWER; + word1--; + if (word1 < 0) { + word1 += LONG_TWO_TO_56_POWER; + word2--; + if (word2 < 0) { + // Underflow. + return 0; + } + } + } + + long longWork = 0; + + int shift = INITIAL_SHIFT; + + if (word2 != 0L) { + + // Skip leading zeroes in word2. + + while (true) { + longWork = (word2 >> shift) & LONG_BYTE_MASK; + if (longWork != 0) { + break; + } + if (shift == 0) { + throw new RuntimeException("Unexpected #1"); + } + shift -= Byte.SIZE; + } + + // Now that we have found real data, emit sign byte if necessary and do negative fixup. + + longWork = (~longWork & LONG_BYTE_MASK); + if (((longWork) & LONG_BYTE_HIGH_BIT_MASK) == 0) { + // Add sign byte since high bit is off. + buffer[byteIndex++] = BYTE_ALL_BITS; + } + + // Invert words. + word2 = ~word2; + word1 = ~word1; + word0 = ~word0; + + // Emit the rest of word2 + while (true) { + buffer[byteIndex++] = (byte) longWork; + if (shift == 0) { + break; + } + shift -= Byte.SIZE; + longWork = (word2 >> shift) & LONG_BYTE_MASK; + } + + shift = INITIAL_SHIFT; + } + + if (byteIndex == 0 && word1 == 0L) { + + // Skip word1, also. + + } else { + + if (byteIndex == 0) { + + // Skip leading zeroes in word1. + + while (true) { + longWork = (word1 >> shift) & LONG_BYTE_MASK; + if (longWork != 0) { + break; + } + if (shift == 0) { + throw new RuntimeException("Unexpected #2"); + } + shift -= Byte.SIZE; + } + + // Now that we have found real data, emit sign byte if necessary and do negative fixup. + + longWork = (~longWork & LONG_BYTE_MASK); + if ((longWork & LONG_BYTE_HIGH_BIT_MASK) == 0) { + // Add sign byte since high bit is off. + buffer[byteIndex++] = BYTE_ALL_BITS; + } + + // Invert words. + word1 = ~word1; + word0 = ~word0; + + } else { + longWork = (word1 >> shift) & LONG_BYTE_MASK; + } + + // Emit the rest of word1 + + while (true) { + buffer[byteIndex++] = (byte) longWork; + if (shift == 0) { + break; + } + shift -= Byte.SIZE; + longWork = (word1 >> shift) & LONG_BYTE_MASK; + } + + shift = INITIAL_SHIFT; + } + + if (byteIndex == 0) { + + // Skip leading zeroes in word0. + + while (true) { + longWork = (word0 >> shift) & LONG_BYTE_MASK; + if (longWork != 0) { + break; + } + if (shift == 0) { + + // All zeroes. + + // -1 special case. Unsigned magnitude 1 - two's compliment adjustment 1 = 0. + buffer[0] = BYTE_ALL_BITS; + return 1; + } + shift -= Byte.SIZE; + } + + // Now that we have found real data, emit sign byte if necessary and do negative fixup. + + longWork = (~longWork & LONG_BYTE_MASK); + if ((longWork & LONG_BYTE_HIGH_BIT_MASK) == 0) { + // Add sign byte since high bit is off. + buffer[byteIndex++] = BYTE_ALL_BITS; + } + + // Invert words. + word0 = ~word0; + + } else { + longWork = (word0 >> shift) & LONG_BYTE_MASK; + } + + // Emit the rest of word0. + while (true) { + buffer[byteIndex++] = (byte) longWork; + if (shift == 0) { + break; + } + shift -= Byte.SIZE; + longWork = (word0 >> shift) & LONG_BYTE_MASK; + } + } + + return byteIndex; + } + + /** + * Convert decimal to BigInteger binary bytes with a serialize scale, similar to the formatScale + * for toFormatString. It adds trailing zeroes when a serializeScale is greater than current + * scale. Or, rounds if scale is less than current scale. + * + * Used by Avro and Parquet serialization. + * + * This emulates the OldHiveDecimal setScale / OldHiveDecimal getInternalStorage() behavior. + * + * @param fastSignum the sign number (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param fastIntegerDigitCount the number of integer digits + * @param fastScale the scale + * @param serializeScale the scale to serialize + * @param scratchLongs a scratch array of longs + * @param buffer the buffer to serialize into + * @return the number of bytes used to serialize the number + */ + public static int fastBigIntegerBytesScaled( + final int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int serializeScale, + long[] scratchLongs, byte[] buffer) { + + // Normally, trailing fractional digits are removed. But to emulate the + // OldHiveDecimal setScale and OldHiveDecimalWritable internalStorage, we need to trailing zeroes + // here. + // + // NOTE: This can cause a decimal that has too many decimal digits (because of trailing zeroes) + // for us to represent. In that case, we punt and convert with a BigInteger alternate + // code. + + if (fastSignum == 0 || serializeScale == fastScale) { + return + fastBigIntegerBytesUnscaled( + fastSignum, fast0, fast1, fast2, + scratchLongs, buffer); + } else if (serializeScale > fastScale) { + + final int scaleUp = serializeScale - fastScale; + final int maxScale = HiveDecimal.MAX_SCALE - fastIntegerDigitCount; + if (serializeScale > maxScale) { + + // We cannot to scaled up decimals that cannot be represented. + // Instead, we use a BigInteger instead. + + BigInteger bigInteger = + fastBigIntegerValueUnscaled( + fastSignum, fast0, fast1, fast2); + + BigInteger bigIntegerScaled = bigInteger.multiply(BIG_INTEGER_TEN.pow(scaleUp)); + byte[] bigIntegerBytesScaled = bigIntegerScaled.toByteArray(); + final int length = bigIntegerBytesScaled.length; + System.arraycopy(bigIntegerBytesScaled, 0, buffer, 0, length); + return length; + } + + FastHiveDecimal fastTemp = new FastHiveDecimal(); + if (!fastScaleUp( + fast0, fast1, fast2, + scaleUp, + fastTemp)) { + throw new RuntimeException("Unexpected"); + } + return + fastBigIntegerBytesUnscaled( + fastSignum, fastTemp.fast0, fastTemp.fast1, fastTemp.fast2, + scratchLongs, buffer); + } else { + + // serializeScale < fastScale. + + FastHiveDecimal fastTemp = new FastHiveDecimal(); + if (!fastRound( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + serializeScale, BigDecimal.ROUND_HALF_UP, + fastTemp)) { + return 0; + } + return + fastBigIntegerBytesUnscaled( + fastSignum, fastTemp.fast0, fastTemp.fast1, fastTemp.fast2, + scratchLongs, buffer); + } + } + + //************************************************************************************************ + // Decimal to Integer conversion. + + private static final int MAX_BYTE_DIGITS = 3; + private static final FastHiveDecimal FASTHIVEDECIMAL_MIN_BYTE_VALUE_MINUS_ONE = + new FastHiveDecimal((long) Byte.MIN_VALUE - 1L); + private static final FastHiveDecimal FASTHIVEDECIMAL_MAX_BYTE_VALUE_PLUS_ONE = + new FastHiveDecimal((long) Byte.MAX_VALUE + 1L); + + private static final int MAX_SHORT_DIGITS = 5; + private static final FastHiveDecimal FASTHIVEDECIMAL_MIN_SHORT_VALUE_MINUS_ONE = + new FastHiveDecimal((long) Short.MIN_VALUE - 1L); + private static final FastHiveDecimal FASTHIVEDECIMAL_MAX_SHORT_VALUE_PLUS_ONE = + new FastHiveDecimal((long) Short.MAX_VALUE + 1L); + + private static final int MAX_INT_DIGITS = 10; + private static final FastHiveDecimal FASTHIVEDECIMAL_MIN_INT_VALUE_MINUS_ONE = + new FastHiveDecimal((long) Integer.MIN_VALUE - 1L); + private static final FastHiveDecimal FASTHIVEDECIMAL_MAX_INT_VALUE_PLUS_ONE = + new FastHiveDecimal((long) Integer.MAX_VALUE + 1L); + + private static final FastHiveDecimal FASTHIVEDECIMAL_MIN_LONG_VALUE = + new FastHiveDecimal(Long.MIN_VALUE); + private static final FastHiveDecimal FASTHIVEDECIMAL_MAX_LONG_VALUE = + new FastHiveDecimal(Long.MAX_VALUE); + private static final int MAX_LONG_DIGITS = + FASTHIVEDECIMAL_MAX_LONG_VALUE.fastIntegerDigitCount; + private static final FastHiveDecimal FASTHIVEDECIMAL_MIN_LONG_VALUE_MINUS_ONE = + new FastHiveDecimal("-9223372036854775809"); + private static final FastHiveDecimal FASTHIVEDECIMAL_MAX_LONG_VALUE_PLUS_ONE = + new FastHiveDecimal("9223372036854775808"); + + private static final BigInteger BIG_INTEGER_UNSIGNED_BYTE_MAX_VALUE = BIG_INTEGER_TWO.pow(Byte.SIZE).subtract(BigInteger.ONE); + private static final BigInteger BIG_INTEGER_UNSIGNED_SHORT_MAX_VALUE = BIG_INTEGER_TWO.pow(Short.SIZE).subtract(BigInteger.ONE); + private static final BigInteger BIG_INTEGER_UNSIGNED_INT_MAX_VALUE = BIG_INTEGER_TWO.pow(Integer.SIZE).subtract(BigInteger.ONE); + private static final BigInteger BIG_INTEGER_UNSIGNED_LONG_MAX_VALUE = BIG_INTEGER_TWO.pow(Long.SIZE).subtract(BigInteger.ONE); + + /** + * Is the decimal value a byte? Range -128 to 127. + * Byte.MIN_VALUE Byte.MAX_VALUE + * + * Emulates testing for no value corruption: + * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().byteValue())) + * + * NOTE: Fractional digits are ignored in the test since fastByteValueClip() will + * remove them (round down). + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param fastIntegerDigitCount the number of integer digits + * @param fastScale the scale of the number + * @return True when fastByteValueClip() will return a correct byte. + */ + public static boolean fastIsByte( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastIntegerDigitCount < MAX_BYTE_DIGITS) { + + // Definitely a byte; most bytes fall here + return true; + + } else if (fastIntegerDigitCount > MAX_BYTE_DIGITS) { + + // Definitely not a byte. + return false; + + } else if (fastScale == 0) { + if (fast1 != 0 || fast2 != 0) { + return false; + } + if (fastSignum == 1) { + return (fast0 <= Byte.MAX_VALUE); + } else { + return (-fast0 >= Byte.MIN_VALUE); + } + } else { + + // We need to work a little harder for our comparison. Note we round down for + // integer conversion so anything below the next min/max will work. + + if (fastSignum == 1) { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MAX_BYTE_VALUE_PLUS_ONE) < 0); + } else { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MIN_BYTE_VALUE_MINUS_ONE) > 0); + } + } + } + + // We use "Clip" in the name because this method will return a corrupted value when + // fastIsByte returns false. + public static byte fastByteValueClip( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastScale == 0) { + if (fast1 == 0 && fast2 == 0) { + if (fastSignum == 1) { + if (fast0 <= Byte.MAX_VALUE) { + return (byte) fast0; + } + } else { + if (-fast0 >= Byte.MIN_VALUE) { + return (byte) -fast0; + }; + } + } + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValueUnscaled( + fastSignum, fast0, fast1, fast2); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_BYTE_MAX_VALUE).byteValue(); + } else { + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + if (fastScale < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[fastScale]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale]; + + result0 = + fast0 / divideFactor + + ((fast1 % divideFactor) * multiplyFactor); + result1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result2 = + fast2 / divideFactor; + + } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + result0 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result1 = + fast2 / divideFactor; + result2 = 0; + + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS; + + result0 = + fast2 / powerOfTenTable[adjustedScaleDown]; + result1 = 0; + result2 = 0; + + } + + if (result1 == 0 && result2 == 0) { + if (fastSignum == 1) { + if (result0 <= Byte.MAX_VALUE) { + return (byte) result0; + } + } else { + if (-result0 >= Byte.MIN_VALUE) { + return (byte) -result0; + }; + } + } + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValueUnscaled( + fastSignum, result0, result1, result2); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_BYTE_MAX_VALUE).byteValue(); + } + } + + /** + + * @return True when shortValue() will return a correct short. + */ + + /** + * Is the decimal value a short? Range -32,768 to 32,767. + * Short.MIN_VALUE Short.MAX_VALUE + * + * Emulates testing for no value corruption: + * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().shortValue())) + * + * NOTE: Fractional digits are ignored in the test since fastShortValueClip() will + * remove them (round down). + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param fastIntegerDigitCount the number of integer digits + * @param fastScale the scale of the number + * @return True when fastShortValueClip() will return a correct short. + */ + public static boolean fastIsShort( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastIntegerDigitCount < MAX_SHORT_DIGITS) { + + // Definitely a short; most shorts fall here + return true; + + } else if (fastIntegerDigitCount > MAX_SHORT_DIGITS) { + + // Definitely not a short. + return false; + + } else if (fastScale == 0) { + if (fast1 != 0 || fast2 != 0) { + return false; + } + if (fastSignum == 1) { + return (fast0 <= Short.MAX_VALUE); + } else { + return (-fast0 >= Short.MIN_VALUE); + } + } else { + + // We need to work a little harder for our comparison. Note we round down for + // integer conversion so anything below the next min/max will work. + + if (fastSignum == 1) { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MAX_SHORT_VALUE_PLUS_ONE) < 0); + } else { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MIN_SHORT_VALUE_MINUS_ONE) > 0); + } + } + } + + // We use "Clip" in the name because this method will return a corrupted value when + // fastIsShort returns false. + public static short fastShortValueClip( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastScale == 0) { + if (fast1 == 0 && fast2 == 0) { + if (fastSignum == 1) { + if (fast0 <= Short.MAX_VALUE) { + return (short) fast0; + } + } else { + if (-fast0 >= Short.MIN_VALUE) { + return (short) -fast0; + }; + } + } + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValueUnscaled( + fastSignum, fast0, fast1, fast2); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_SHORT_MAX_VALUE).shortValue(); + } else { + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + if (fastScale < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[fastScale]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale]; + + result0 = + fast0 / divideFactor + + ((fast1 % divideFactor) * multiplyFactor); + result1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result2 = + fast2 / divideFactor; + + } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + result0 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result1 = + fast2 / divideFactor; + result2 = 0; + + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS; + + result0 = + fast2 / powerOfTenTable[adjustedScaleDown]; + result1 = 0; + result2 = 0; + + } + + if (result1 == 0 && result2 == 0) { + if (fastSignum == 1) { + if (result0 <= Short.MAX_VALUE) { + return (short) result0; + } + } else { + if (-result0 >= Short.MIN_VALUE) { + return (short) -result0; + }; + } + } + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValueUnscaled( + fastSignum, result0, result1, result2); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_SHORT_MAX_VALUE).shortValue(); + } + } + + /** + * Is the decimal value a int? Range -2,147,483,648 to 2,147,483,647. + * Integer.MIN_VALUE Integer.MAX_VALUE + * + * Emulates testing for no value corruption: + * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().intValue())) + * + * NOTE: Fractional digits are ignored in the test since fastIntValueClip() will + * remove them (round down). + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param fastIntegerDigitCount the number of integer digits + * @param fastScale the scale of the number + * @return True when fastIntValueClip() will return a correct int. + */ + public static boolean fastIsInt( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastIntegerDigitCount < MAX_INT_DIGITS) { + + // Definitely a int; most ints fall here + return true; + + } else if (fastIntegerDigitCount > MAX_INT_DIGITS) { + + // Definitely not an int. + return false; + + } else if (fastScale == 0) { + if (fast1 != 0 || fast2 != 0) { + return false; + } + if (fastSignum == 1) { + return (fast0 <= Integer.MAX_VALUE); + } else { + return (-fast0 >= Integer.MIN_VALUE); + } + } else { + + // We need to work a little harder for our comparison. Note we round down for + // integer conversion so anything below the next min/max will work. + + if (fastSignum == 1) { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MAX_INT_VALUE_PLUS_ONE) < 0); + } else { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MIN_INT_VALUE_MINUS_ONE) > 0); + } + } + } + + // We use "Clip" in the name because this method will return a corrupted value when + // fastIsInt returns false. + public static int fastIntValueClip( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastScale == 0) { + if (fast1 == 0 && fast2 == 0) { + if (fastSignum == 1) { + if (fast0 <= Integer.MAX_VALUE) { + return (int) fast0; + } + } else { + if (-fast0 >= Integer.MIN_VALUE) { + return (int) -fast0; + }; + } + } + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValueUnscaled( + fastSignum, fast0, fast1, fast2); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_INT_MAX_VALUE).intValue(); + } else { + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + if (fastScale < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[fastScale]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale]; + + result0 = + fast0 / divideFactor + + ((fast1 % divideFactor) * multiplyFactor); + result1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result2 = + fast2 / divideFactor; + + } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + result0 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result1 = + fast2 / divideFactor; + result2 = 0; + + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS; + + result0 = + fast2 / powerOfTenTable[adjustedScaleDown]; + result1 = 0; + result2 = 0; + + } + + if (result1 == 0 && result2 == 0) { + if (fastSignum == 1) { + if (result0 <= Integer.MAX_VALUE) { + return (int) result0; + } + } else { + if (-result0 >= Integer.MIN_VALUE) { + return (int) -result0; + }; + } + } + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValueUnscaled( + fastSignum, result0, result1, result2); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_INT_MAX_VALUE).intValue(); + } + } + + /** + * Is the decimal value a long? Range -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807. + * Long.MIN_VALUE Long.MAX_VALUE + * + * Emulates testing for no value corruption: + * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().longValue())) + * + * NOTE: Fractional digits are ignored in the test since fastLongValueClip() will + * remove them (round down). + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param fastIntegerDigitCount the number of integer digits + * @param fastScale the scale of the number + * @return True when fastLongValueClip() will return a correct long. + */ + public static boolean fastIsLong( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastIntegerDigitCount < MAX_LONG_DIGITS) { + + // Definitely a long; most longs fall here + return true; + + } else if (fastIntegerDigitCount > MAX_LONG_DIGITS) { + + // Definitely not a long. + return false; + + } else if (fastScale == 0) { + + // From the above checks, we know fast2 is zero. + + if (fastSignum == 1) { + FastHiveDecimal max = FASTHIVEDECIMAL_MAX_LONG_VALUE; + if (fast1 > max.fast1 || (fast1 == max.fast1 && fast0 > max.fast0)) { + return false; + } + return true; + } else { + FastHiveDecimal min = FASTHIVEDECIMAL_MIN_LONG_VALUE; + if (fast1 > min.fast1 || (fast1 == min.fast1 && fast0 > min.fast0)) { + return false; + } + return true; + } + + } else { + + // We need to work a little harder for our comparison. Note we round down for + // integer conversion so anything below the next min/max will work. + + if (fastSignum == 1) { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MAX_LONG_VALUE_PLUS_ONE) < 0); + } else { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MIN_LONG_VALUE_MINUS_ONE) > 0); + } + } + } + + // We use "Clip" in the name because this method will return a corrupted value when + // fastIsLong returns false. + public static long fastLongValueClip( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + if (fastSignum == 0) { + return 0; + } + + if (fastScale == 0) { + // Do first comparison as unsigned. + if (fastCompareTo( + 1, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MAX_LONG_VALUE) <= 0) { + if (fastSignum == 1) { + return + fast1 * MULTIPLER_LONGWORD_DECIMAL + + fast0; + } else { + return + -(fast1 * MULTIPLER_LONGWORD_DECIMAL + + fast0); + } + } if (fastEquals( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MIN_LONG_VALUE)) { + return Long.MIN_VALUE; + } else { + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValueUnscaled( + fastSignum, fast0, fast1, fast2); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_LONG_MAX_VALUE).longValue(); + } + } else { + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + if (fastScale < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[fastScale]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale]; + + result0 = + fast0 / divideFactor + + ((fast1 % divideFactor) * multiplyFactor); + result1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result2 = + fast2 / divideFactor; + + } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + result0 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result1 = + fast2 / divideFactor; + result2 = 0; + + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS; + + result0 = + fast2 / powerOfTenTable[adjustedScaleDown]; + result1 = 0; + result2 = 0; + + } + + // Do first comparison as UNSIGNED. + if (fastCompareTo( + 1, result0, result1, result2, /* fastScale */ 0, + FASTHIVEDECIMAL_MAX_LONG_VALUE) <= 0) { + if (fastSignum == 1) { + return + result1 * MULTIPLER_LONGWORD_DECIMAL + + result0; + } else { + return + -(result1 * MULTIPLER_LONGWORD_DECIMAL + + result0); + } + } if (fastEquals( + fastSignum, result0, result1, result2, /* fastScale */ 0, + FASTHIVEDECIMAL_MIN_LONG_VALUE)) { + + // SIGNED comparison to Long.MIN_VALUE decimal. + return Long.MIN_VALUE; + } else { + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValueUnscaled( + fastSignum, result0, result1, result2); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_LONG_MAX_VALUE).longValue(); + } + } + } + + //************************************************************************************************ + // Decimal to Non-Integer conversion. + + public static float fastFloatValue( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + if (fastSignum == 0) { + return 0; + } + BigDecimal bigDecimal = fastBigDecimalValue( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale); + return bigDecimal.floatValue(); + } + + public static double fastDoubleValue( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + if (fastSignum == 0) { + return 0; + } + + // CONSIDER: Looked at the possibility of faster decimal to double conversion by using some + // of their lower level logic that extracts the various parts out of a double. + // The difficulty is Java's rounding rules are byzantine. + + BigDecimal bigDecimal = fastBigDecimalValue( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale); + return bigDecimal.doubleValue(); + } + + /** + * Get a BigInteger representing the decimal's digits without a dot. + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param fastIntegerDigitCount the number of integer digits + * @param fastScale the scale of the number + * @param fastSerializationScale the scale to serialize + * @return Returns a signed BigInteger. + */ + public static BigInteger fastBigIntegerValue( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int fastSerializationScale) { + if (fastSerializationScale != -1) { + return + fastBigIntegerValueScaled( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + fastSerializationScale); + } else { + return + fastBigIntegerValueUnscaled( + fastSignum, fast0, fast1, fast2); + } + } + + public static BigInteger fastBigIntegerValueUnscaled( + int fastSignum, long fast0, long fast1, long fast2) { + + if (fastSignum == 0) { + return BigInteger.ZERO; + } + BigInteger result; + if (fast2 == 0) { + if (fast1 == 0) { + result = + BigInteger.valueOf(fast0); + } else { + result = + BigInteger.valueOf(fast0).add( + BigInteger.valueOf(fast1).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER)); + } + } else { + result = + BigInteger.valueOf(fast0).add( + BigInteger.valueOf(fast1).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER)).add( + BigInteger.valueOf(fast2).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER_2X)); + } + + return (fastSignum == 1 ? result : result.negate()); + } + + public static BigInteger fastBigIntegerValueScaled( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int fastSerializationScale) { + + // Use the serialization scale and create a BigInteger with trailing zeroes (or + // round the decimal) if necessary. + // + // Since we are emulating old behavior and recommending the use of HiveDecimal.bigIntegerBytesScaled + // instead just do it the slow way. Get the BigDecimal.setScale value and return the + // BigInteger. + // + BigDecimal bigDecimal = + fastBigDecimalValue( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale); + bigDecimal = bigDecimal.setScale(fastSerializationScale, RoundingMode.HALF_UP); + return bigDecimal.unscaledValue(); + } + + /** + * Return a BigDecimal representing the decimal. The BigDecimal class is able to accurately + * represent the decimal. + * + * NOTE: We are not representing our decimal as BigDecimal now as OldHiveDecimal did, so this + * is now slower. + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param fastIntegerDigitCount the number of integer digits + * @param fastScale the scale of the number + * @return the BigDecimal equivalent + */ + public static BigDecimal fastBigDecimalValue( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + BigInteger unscaledValue = + fastBigIntegerValueUnscaled( + fastSignum, fast0, fast1, fast2); + return new BigDecimal(unscaledValue, fastScale); + } + + //************************************************************************************************ + // Decimal Comparison. + + public static int fastCompareTo( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftScale, + FastHiveDecimal fastRight) { + + return + fastCompareTo( + leftSignum, leftFast0, leftFast1, leftFast2, + leftScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastScale); + } + + private static int doCompareToSameScale( + int signum, + long leftFast0, long leftFast1, long leftFast2, + long rightFast0, long rightFast1, long rightFast2) { + + if (leftFast0 == rightFast0 && leftFast1 == rightFast1 && leftFast2 == rightFast2) { + return 0; + } + if (leftFast2 < rightFast2) { + return -signum; + } else if (leftFast2 > rightFast2) { + return signum; + } + if (leftFast1 < rightFast1) { + return -signum; + } else if (leftFast1 > rightFast1){ + return signum; + } + return (leftFast0 < rightFast0 ? -signum : signum); + } + + public static int fastCompareTo( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightScale) { + + if (leftSignum == 0 && rightSignum == 0) { + return 0; + } + + // Optimization copied from BigDecimal. + int signDiff = leftSignum - rightSignum; + if (signDiff != 0) { + return (signDiff > 0 ? 1 : -1); + } + + // We are here when the left and right are non-zero and have the same sign. + + if (leftScale == rightScale) { + + return doCompareToSameScale( + leftSignum, + leftFast0, leftFast1, leftFast2, + rightFast0, rightFast1, rightFast2); + + } else { + + // How do we handle different scales? + + // We at least know they are not equal. The one with the larger scale has non-zero digits + // below the other's scale (since the scale does not include trailing zeroes). + + // For comparison purposes, we can scale away those digits. And, we can not scale up since + // that could overflow. + + // Use modified portions of doFastScaleDown code here since we do not want to allocate a + // temporary FastHiveDecimal object. + + long compare0; + long compare1; + long compare2; + int scaleDown; + if (leftScale < rightScale) { + + // Scale down right and compare. + scaleDown = rightScale - leftScale; + + // Adjust all longs using power 10 division/remainder. + + if (scaleDown < LONGWORD_DECIMAL_DIGITS) { + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[scaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; + + compare0 = + rightFast0 / divideFactor + + ((rightFast1 % divideFactor) * multiplyFactor); + compare1 = + rightFast1 / divideFactor + + ((rightFast2 % divideFactor) * multiplyFactor); + compare2 = + rightFast2 / divideFactor; + } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) { + // Throw away lowest word. + + final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + compare0 = + rightFast1 / divideFactor + + ((rightFast2 % divideFactor) * multiplyFactor); + compare1 = + rightFast2 / divideFactor; + compare2 = 0; + } else { + // Throw away middle and lowest words. + + final int adjustedScaleDown = scaleDown - TWO_X_LONGWORD_DECIMAL_DIGITS; + + compare0 = + rightFast2 / powerOfTenTable[adjustedScaleDown]; + compare1 = 0; + compare2 = 0; + } + + if (leftFast0 == compare0 && leftFast1 == compare1 && leftFast2 == compare2) { + // Return less than because of right's digits below left's scale. + return -leftSignum; + } + if (leftFast2 < compare2) { + return -leftSignum; + } else if (leftFast2 > compare2) { + return leftSignum; + } + if (leftFast1 < compare1) { + return -leftSignum; + } else if (leftFast1 > compare1){ + return leftSignum; + } + return (leftFast0 < compare0 ? -leftSignum : leftSignum); + + } else { + + // Scale down left and compare. + scaleDown = leftScale - rightScale; + + // Adjust all longs using power 10 division/remainder. + + if (scaleDown < LONGWORD_DECIMAL_DIGITS) { + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[scaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; + + compare1 = + leftFast1 / divideFactor + + ((leftFast2 % divideFactor) * multiplyFactor); + compare0 = + leftFast0 / divideFactor + + ((leftFast1 % divideFactor) * multiplyFactor); + compare2 = + leftFast2 / divideFactor; + } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) { + // Throw away lowest word. + + final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + compare0 = + leftFast1 / divideFactor + + ((leftFast2 % divideFactor) * multiplyFactor); + compare1 = + leftFast2 / divideFactor; + compare2 = 0; + } else { + // Throw away middle and lowest words. + + final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS; + + compare0 = + leftFast2 / powerOfTenTable[adjustedScaleDown]; + compare1 = 0; + compare2 = 0; + } + + if (compare0 == rightFast0 && compare1 == rightFast1 && compare2 == rightFast2) { + // Return greater than because of left's digits below right's scale. + return leftSignum; + } + if (compare2 < rightFast2) { + return -leftSignum; + } else if (compare2 > rightFast2) { + return leftSignum; + } + if (compare1 < rightFast1) { + return -leftSignum; + } else if (compare1 > rightFast1){ + return leftSignum; + } + return (compare0 < rightFast0 ? -leftSignum : leftSignum); + + } + } + } + + public static boolean fastEquals( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftScale, + FastHiveDecimal fastRight) { + + if (leftSignum == 0) { + return (fastRight.fastSignum == 0); + } + if (leftSignum != fastRight.fastSignum) { + return false; + } + if (leftScale != fastRight.fastScale) { + // We know they are not equal because the one with the larger scale has non-zero digits + // below the other's scale (since the scale does not include trailing zeroes). + return false; + } + return ( + leftFast0 == fastRight.fast0 && leftFast1 == fastRight.fast1 && leftFast2 == fastRight.fast2); + } + + public static boolean fastEquals( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightScale) { + + if (leftSignum == 0) { + return (rightSignum == 0); + } + if (leftSignum != rightSignum) { + return false; + } + if (leftScale != rightScale) { + // We know they are not equal because the one with the larger scale has non-zero digits + // below the other's scale (since the scale does not include trailing zeroes). + return false; + } + return ( + leftFast0 == rightFast0 && leftFast1 == rightFast1 && leftFast2 == rightFast2); + } + + private static int doCalculateNewFasterHashCode( + int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) { + + long longHashCode; + + long key = fast0; + + // Hash code logic from original calculateLongHashCode + + key = (~key) + (key << 21); // key = (key << 21) - key - 1; + key = key ^ (key >>> 24); + key = (key + (key << 3)) + (key << 8); // key * 265 + key = key ^ (key >>> 14); + key = (key + (key << 2)) + (key << 4); // key * 21 + key = key ^ (key >>> 28); + key = key + (key << 31); + + longHashCode = key; + + key = fast1; + + key = (~key) + (key << 21); // key = (key << 21) - key - 1; + key = key ^ (key >>> 24); + key = (key + (key << 3)) + (key << 8); // key * 265 + key = key ^ (key >>> 14); + key = (key + (key << 2)) + (key << 4); // key * 21 + key = key ^ (key >>> 28); + key = key + (key << 31); + + longHashCode ^= key; + + key = fast2; + + key = (~key) + (key << 21); // key = (key << 21) - key - 1; + key = key ^ (key >>> 24); + key = (key + (key << 3)) + (key << 8); // key * 265 + key = key ^ (key >>> 14); + key = (key + (key << 2)) + (key << 4); // key * 21 + key = key ^ (key >>> 28); + key = key + (key << 31); + + longHashCode ^= key; + + key = fastSignum; + + key = (~key) + (key << 21); // key = (key << 21) - key - 1; + key = key ^ (key >>> 24); + key = (key + (key << 3)) + (key << 8); // key * 265 + key = key ^ (key >>> 14); + key = (key + (key << 2)) + (key << 4); // key * 21 + key = key ^ (key >>> 28); + key = key + (key << 31); + + longHashCode ^= key; + + key = fastIntegerDigitCount; + + key = (~key) + (key << 21); // key = (key << 21) - key - 1; + key = key ^ (key >>> 24); + key = (key + (key << 3)) + (key << 8); // key * 265 + key = key ^ (key >>> 14); + key = (key + (key << 2)) + (key << 4); // key * 21 + key = key ^ (key >>> 28); + key = key + (key << 31); + + longHashCode ^= key; + + key = fastScale; + + key = (~key) + (key << 21); // key = (key << 21) - key - 1; + key = key ^ (key >>> 24); + key = (key + (key << 3)) + (key << 8); // key * 265 + key = key ^ (key >>> 14); + key = (key + (key << 2)) + (key << 4); // key * 21 + key = key ^ (key >>> 28); + key = key + (key << 31); + + longHashCode ^= key; + + return (int) longHashCode; + } + + private static final int ZERO_NEW_FASTER_HASH_CODE = doCalculateNewFasterHashCode(0, 0, 0, 0, 0, 0); + + /** + * Hash code based on (new) decimal representation. + * + * Faster than fastHashCode(). + * + * Used by map join and other Hive internal purposes where performance is important. + * + * IMPORTANT: See comments for fastHashCode(), too. + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param fastIntegerDigitCount the number of integer digits + * @param fastScale the scale of the number + * @return the hash code + */ + public static int fastNewFasterHashCode( + int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) { + if (fastSignum == 0) { + return ZERO_NEW_FASTER_HASH_CODE; + } + int hashCode = doCalculateNewFasterHashCode(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + return hashCode; + } + + /** + * This is the original hash code as returned by OldHiveDecimal. + * + * We need this when the OldHiveDecimal hash code has been exposed and and written or affected + * how data is written. + * + * This method supports compatibility. + * + * Examples: bucketing and the Hive hash() function. + * + * NOTE: It is necessary to create a BigDecimal object and use its hash code, so this method is + * slow. + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param fastIntegerDigitCount the number of integer digits + * @param fastScale the scale of the number + * @return the hash code + */ + public static int fastHashCode( + int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) { + + // OldHiveDecimal returns the hash code of its internal BigDecimal. Our TestHiveDecimal + // verifies the OldHiveDecimal.bigDecimalValue() matches (new) HiveDecimal.bigDecimalValue(). + + BigDecimal bigDecimal = + fastBigDecimalValue( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + + return bigDecimal.hashCode(); + } + + //************************************************************************************************ + // Decimal Math. + + public static boolean fastScaleByPowerOfTen( + FastHiveDecimal fastDec, + int power, + FastHiveDecimal fastResult) { + return fastScaleByPowerOfTen( + fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2, + fastDec.fastIntegerDigitCount, fastDec.fastScale, + power, + fastResult); + } + + // NOTE: power can be positive or negative. + // NOTE: e.g. power = 2 is effectively multiply by 10^2 + // NOTE: and power = -3 is multiply by 10^-3 + public static boolean fastScaleByPowerOfTen( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int power, + FastHiveDecimal fastResult) { + + if (fastSignum == 0) { + fastResult.fastReset(); + return true; + } + if (power == 0) { + fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + return true; + } + + final int absPower = Math.abs(power); + + if (power > 0) { + + int integerRoom; + int fractionalRoom; + if (fastIntegerDigitCount > 0) { + + // Is there integer room above? + + integerRoom = HiveDecimal.MAX_PRECISION - fastIntegerDigitCount; + if (integerRoom < power) { + return false; + } + fastResult.fastSignum = fastSignum; + if (fastScale <= power) { + + // All fractional digits become integer digits. + final int scaleUp = power - fastScale; + if (scaleUp > 0) { + if (!fastScaleUp( + fast0, fast1, fast2, + scaleUp, + fastResult)) { + throw new RuntimeException("Unexpected"); + } + } else { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } + fastResult.fastIntegerDigitCount = fastIntegerDigitCount + fastScale + scaleUp; + fastResult.fastScale = 0; + + } else { + + // Only a scale adjustment is needed. + fastResult.fastSet( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount + power, fastScale - power); + } + } else { + + // How much can the fraction be moved up? + + final int rawPrecision = fastRawPrecision(fastSignum, fast0, fast1, fast2); + final int zeroesBelowDot = fastScale - rawPrecision; + + // Our limit is max precision integer digits + "leading" zeros below the dot. + // E.g. 0.00021 has 3 zeroes below the dot. + // + if (power > HiveDecimal.MAX_PRECISION + zeroesBelowDot) { + + // Fractional part powered up too high. + return false; + } + + final int newIntegerDigitCount = Math.max(0, power - zeroesBelowDot); + if (newIntegerDigitCount > rawPrecision) { + + fastResult.fastSignum = fastSignum; + final int scaleUp = newIntegerDigitCount - rawPrecision; + if (!fastScaleUp( + fast0, fast1, fast2, + scaleUp, + fastResult)) { + throw new RuntimeException("Unexpected"); + } + fastResult.fastIntegerDigitCount = newIntegerDigitCount; + fastResult.fastScale = 0; + } else { + final int newScale = Math.max(0, fastScale - power); + fastResult.fastSet(fastSignum, fast0, fast1, fast2, newIntegerDigitCount, newScale); + } + } + + } else if (fastScale + absPower <= HiveDecimal.MAX_SCALE) { + + // Negative power with range -- adjust the scale. + + final int newScale = fastScale + absPower; + final int newIntegerDigitCount = Math.max(0, fastIntegerDigitCount - absPower); + + final int trailingZeroCount = + fastTrailingDecimalZeroCount( + fast0, fast1, fast2, + newIntegerDigitCount, newScale); + if (trailingZeroCount > 0) { + fastResult.fastSignum = fastSignum; + doFastScaleDown( + fast0, fast1, fast2, + trailingZeroCount, fastResult); + fastResult.fastScale = newScale - trailingZeroCount; + fastResult.fastIntegerDigitCount = newIntegerDigitCount; + } else { + fastResult.fastSet(fastSignum, fast0, fast1, fast2, + newIntegerDigitCount, newScale); + } + } else { + + // fastScale + absPower > HiveDecimal.MAX_SCALE + + // Look at getting rid of fractional digits that will now be below HiveDecimal.MAX_SCALE. + + final int scaleDown = fastScale + absPower - HiveDecimal.MAX_SCALE; + + if (scaleDown < HiveDecimal.MAX_SCALE) { + if (!fastRoundFractionalHalfUp( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult)) { + // Overflow. + return false; + } + if (fastResult.fastSignum != 0) { + + fastResult.fastScale = HiveDecimal.MAX_SCALE; + fastResult.fastIntegerDigitCount = + Math.max(0, fastRawPrecision(fastResult) - fastResult.fastScale); + + final int trailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (trailingZeroCount > 0) { + doFastScaleDown( + fastResult, + trailingZeroCount, + fastResult); + fastResult.fastScale -= trailingZeroCount; + } + } + } else { + // All precision has been lost -- result is 0. + fastResult.fastReset(); + } + } + + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + + return true; + } + + //************************************************************************************************ + // Decimal Rounding. + + public static boolean doFastRound( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int roundPower, + int roundingMode, + FastHiveDecimal fastResult) { + + if (fastSignum == 0) { + + // Zero result. + fastResult.fastReset(); + return true; + } else if (fastScale == roundPower) { + + // The roundPower same as scale means all zeroes below round point. + + fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + return true; + } + + if (roundPower > fastScale) { + + // We pretend to add trailing zeroes, EVEN WHEN it would exceed the HiveDecimal.MAX_PRECISION. + + // Copy current value; do not change current scale. + fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + } else if (roundPower < 0) { + + // roundPower < 0 + // + // Negative scale means we start rounding integer digits. + // + // The result will integer result will have at least abs(roundPower) trailing digits. + // + // Examples where the 'r's show the rounding digits: + // + // round(12500, -3) = 13000 // BigDecimal.ROUND_HALF_UP + // rrr + // + // Or, ceiling(12400.8302, -2) = 12500 // BigDecimal.ROUND_CEILING + // rr rrrr + // + // Notice that any fractional digits will be gone in the result. + // + switch (roundingMode) { + case BigDecimal.ROUND_DOWN: + if (!fastRoundIntegerDown( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + break; + case BigDecimal.ROUND_UP: + if (!fastRoundIntegerUp( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + break; + case BigDecimal.ROUND_FLOOR: + // Round towards negative infinity. + if (fastSignum == 1) { + if (!fastRoundIntegerDown( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + } else { + if (!fastRoundIntegerUp( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) { + return false; + } + } + break; + case BigDecimal.ROUND_CEILING: + // Round towards positive infinity. + if (fastSignum == 1) { + if (!fastRoundIntegerUp( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) { + return false; + } + } else { + if (!fastRoundIntegerDown( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + } + break; + case BigDecimal.ROUND_HALF_UP: + if (!fastRoundIntegerHalfUp( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) { + return false; + } + break; + case BigDecimal.ROUND_HALF_EVEN: + if (!fastRoundIntegerHalfEven( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) { + return false; + } + break; + default: + throw new RuntimeException("Unsupported rounding mode " + roundingMode); + } + + // The fastRoundInteger* methods remove all fractional digits, set fastIntegerDigitCount, and + // set fastScale to 0. + return true; + + } else { + + // roundPower < fastScale + + // Do rounding of fractional digits. + final int scaleDown = fastScale - roundPower; + switch (roundingMode) { + case BigDecimal.ROUND_DOWN: + fastRoundFractionalDown( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult); + break; + case BigDecimal.ROUND_UP: + if (!fastRoundFractionalUp( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult)) { + return false; + } + break; + case BigDecimal.ROUND_FLOOR: + // Round towards negative infinity. + if (fastSignum == 1) { + fastRoundFractionalDown( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult); + } else { + if (!fastRoundFractionalUp( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult)) { + return false; + } + } + break; + case BigDecimal.ROUND_CEILING: + // Round towards positive infinity. + if (fastSignum == 1) { + if (!fastRoundFractionalUp( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult)) { + return false; + } + } else { + fastRoundFractionalDown( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult); + } + break; + case BigDecimal.ROUND_HALF_UP: + if (!fastRoundFractionalHalfUp( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult)) { + return false; + } + break; + case BigDecimal.ROUND_HALF_EVEN: + if (!fastRoundFractionalHalfEven( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult)) { + return false; + } + break; + default: + throw new RuntimeException("Unsupported rounding mode " + roundingMode); + } + if (fastResult.fastSignum == 0) { + fastResult.fastScale = 0; + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + } else { + final int rawPrecision = fastRawPrecision(fastResult); + fastResult.fastIntegerDigitCount = Math.max(0, rawPrecision - roundPower); + fastResult.fastScale = roundPower; + + // Trim trailing zeroes and re-adjust scale. + final int trailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (trailingZeroCount > 0) { + doFastScaleDown( + fastResult, + trailingZeroCount, + fastResult); + fastResult.fastScale -= trailingZeroCount; + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + } + } + } + + return true; + } + + public static boolean fastRound( + FastHiveDecimal fastDec, + int newScale, int roundingMode, + FastHiveDecimal fastResult) { + return fastRound( + fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2, + fastDec.fastIntegerDigitCount, fastDec.fastScale, + newScale, roundingMode, + fastResult); + } + + public static boolean fastRound( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int newScale, int roundingMode, + FastHiveDecimal fastResult) { + return doFastRound( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + newScale, roundingMode, + fastResult); + } + + private static boolean isRoundPortionAllZeroes( + long fast0, long fast1, long fast2, + int roundingPoint) { + + boolean isRoundPortionAllZeroes; + if (roundingPoint < LONGWORD_DECIMAL_DIGITS) { + + // Lowest word gets integer rounding. + + // Factor includes scale. + final long roundPointFactor = powerOfTenTable[roundingPoint]; + + isRoundPortionAllZeroes = (fast0 % roundPointFactor == 0); + + } else if (roundingPoint < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Middle word gets integer rounding. + + final int adjustedRoundingPoint = roundingPoint - LONGWORD_DECIMAL_DIGITS; + + if (adjustedRoundingPoint == 0) { + isRoundPortionAllZeroes = (fast0 == 0); + } else { + + // Factor includes scale. + final long roundPointFactor = powerOfTenTable[adjustedRoundingPoint]; + + final long roundPortion = fast1 % roundPointFactor; + isRoundPortionAllZeroes = (fast0 == 0 && roundPortion == 0); + } + + } else { + + // High word gets integer rounding. + + final int adjustedRoundingPoint = roundingPoint - TWO_X_LONGWORD_DECIMAL_DIGITS; + + if (adjustedRoundingPoint == 0) { + isRoundPortionAllZeroes = (fast0 == 0 && fast1 == 0); + } else { + + // Factor includes scale. + final long roundPointFactor = powerOfTenTable[adjustedRoundingPoint]; + + final long roundPortion = fast2 % roundPointFactor; + isRoundPortionAllZeroes = (fast0 == 0 && fast1 == 0 && roundPortion == 0); + } + } + return isRoundPortionAllZeroes; + } + + private static boolean isRoundPortionHalfUp( + long fast0, long fast1, long fast2, + int roundingPoint) { + + boolean isRoundPortionHalfUp; + if (roundingPoint < LONGWORD_DECIMAL_DIGITS) { + + // Lowest word gets integer rounding. + + // Divide down just before round point to get round digit. + final long withRoundDigit = fast0 / powerOfTenTable[roundingPoint - 1]; + final long roundDigit = withRoundDigit % 10; + + isRoundPortionHalfUp = (roundDigit >= 5); + + } else if (roundingPoint < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Middle word gets integer rounding. + + final int adjustedRoundingPoint = roundingPoint - LONGWORD_DECIMAL_DIGITS; + + long roundDigit; + if (adjustedRoundingPoint == 0) { + // Grab round digit from lowest word. + roundDigit = fast0 / (MULTIPLER_LONGWORD_DECIMAL / 10); + } else { + // Divide down just before scaleDown to get round digit. + final long withRoundDigit = fast1 / powerOfTenTable[adjustedRoundingPoint - 1]; + roundDigit = withRoundDigit % 10; + } + + isRoundPortionHalfUp = (roundDigit >= 5); + + } else { + + // High word gets integer rounding. + + final int adjustedRoundingPoint = roundingPoint - TWO_X_LONGWORD_DECIMAL_DIGITS; + + long roundDigit; + if (adjustedRoundingPoint == 0) { + // Grab round digit from middle word. + roundDigit = fast1 / (MULTIPLER_LONGWORD_DECIMAL / 10); + } else { + // Divide down just before scaleDown to get round digit. + final long withRoundDigit = fast2 / powerOfTenTable[adjustedRoundingPoint - 1]; + roundDigit = withRoundDigit % 10; + } + + isRoundPortionHalfUp = (roundDigit >= 5); + + } + return isRoundPortionHalfUp; + } + + private static boolean isRoundPortionHalfEven( + long fast0, long fast1, long fast2, + int roundingPoint) { + + boolean isRoundPortionHalfEven; + if (roundingPoint < LONGWORD_DECIMAL_DIGITS) { + + // Lowest word gets integer rounding. + + // Divide down just before scaleDown to get round digit. + final long roundDivisor = powerOfTenTable[roundingPoint - 1]; + final long withRoundDigit = fast0 / roundDivisor; + final long roundDigit = withRoundDigit % 10; + final long fast0Scaled = withRoundDigit / 10; + + if (roundDigit > 5) { + isRoundPortionHalfEven = true; + } else if (roundDigit == 5) { + boolean exactlyOneHalf; + if (roundingPoint - 1 == 0) { + // Fraction below 0.5 is implicitly 0. + exactlyOneHalf = true; + } else { + exactlyOneHalf = (fast0 % roundDivisor == 0); + } + + // When fraction is exactly 0.5 and lowest new digit is odd, go towards even. + if (exactlyOneHalf) { + isRoundPortionHalfEven = (fast0Scaled % 2 == 1); + } else { + isRoundPortionHalfEven = true; + } + } else { + isRoundPortionHalfEven = false; + } + + } else if (roundingPoint < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Middle word gets integer rounding. + + final int adjustedRoundingPoint = roundingPoint - LONGWORD_DECIMAL_DIGITS; + + long roundDigit; + long fast1Scaled; + if (adjustedRoundingPoint == 0) { + // Grab round digit from lowest word. + final long roundDivisor = MULTIPLER_LONGWORD_DECIMAL / 10; + roundDigit = fast0 / roundDivisor; + fast1Scaled = fast1; + if (roundDigit > 5) { + isRoundPortionHalfEven = true; + } else if (roundDigit == 5) { + boolean exactlyOneHalf = (fast0 % roundDivisor == 0); + + // When fraction is exactly 0.5 and lowest new digit is odd, go towards even. + if (exactlyOneHalf) { + isRoundPortionHalfEven = (fast1Scaled % 2 == 1); + } else { + isRoundPortionHalfEven = true; + } + } else { + isRoundPortionHalfEven = false; + } + } else { + // Divide down just before scaleDown to get round digit. + final long roundDivisor = powerOfTenTable[adjustedRoundingPoint - 1]; + final long withRoundDigit = fast1 / roundDivisor; + roundDigit = withRoundDigit % 10; + fast1Scaled = withRoundDigit / 10; + if (roundDigit > 5) { + isRoundPortionHalfEven = true; + } else if (roundDigit == 5) { + boolean exactlyOneHalf; + if (adjustedRoundingPoint - 1 == 0) { + // Just examine the lower word. + exactlyOneHalf = (fast0 == 0); + } else { + exactlyOneHalf = (fast0 == 0 && fast1 % roundDivisor == 0); + } + + // When fraction is exactly 0.5 and lowest new digit is odd, go towards even. + if (exactlyOneHalf) { + isRoundPortionHalfEven = (fast1Scaled % 2 == 1); + } else { + isRoundPortionHalfEven = true; + } + } else { + isRoundPortionHalfEven = false; + } + } + + } else { + + // High word gets integer rounding. + + final int adjustedRoundingPoint = roundingPoint - TWO_X_LONGWORD_DECIMAL_DIGITS; + + long roundDigit; + long fast2Scaled; + if (adjustedRoundingPoint == 0) { + // Grab round digit from middle word. + final long roundDivisor = MULTIPLER_LONGWORD_DECIMAL / 10; + roundDigit = fast1 / roundDivisor; + fast2Scaled = fast2; + if (roundDigit > 5) { + isRoundPortionHalfEven = true; + } else if (roundDigit == 5) { + boolean exactlyOneHalf = (fast1 % roundDivisor == 0 && fast0 == 0); + + // When fraction is exactly 0.5 and lowest new digit is odd, go towards even. + if (exactlyOneHalf) { + isRoundPortionHalfEven = (fast2Scaled % 2 == 1); + } else { + isRoundPortionHalfEven = true; + } + } else { + isRoundPortionHalfEven = false; + } + } else { + // Divide down just before scaleDown to get round digit. + final long roundDivisor = powerOfTenTable[adjustedRoundingPoint - 1]; + final long withRoundDigit = fast2 / roundDivisor; + roundDigit = withRoundDigit % 10; + fast2Scaled = withRoundDigit / 10; + if (roundDigit > 5) { + isRoundPortionHalfEven = true; + } else if (roundDigit == 5) { + boolean exactlyOneHalf; + if (adjustedRoundingPoint - 1 == 0) { + // Just examine the middle and lower words. + exactlyOneHalf = (fast1 == 0 && fast0 == 0); + } else { + exactlyOneHalf = (fast2 % roundDivisor == 0 && fast1 == 0 && fast0 == 0); + } + + // When fraction is exactly 0.5 and lowest new digit is odd, go towards even. + if (exactlyOneHalf) { + isRoundPortionHalfEven = (fast2Scaled % 2 == 1); + } else { + isRoundPortionHalfEven = true; + } + } else { + isRoundPortionHalfEven = false; + } + } + } + return isRoundPortionHalfEven; + } + + private static void doClearRoundIntegerPortionAndAddOne( + long fast0, long fast1, long fast2, + int absRoundPower, + FastHiveDecimal fastResult) { + + long result0; + long result1; + long result2; + + if (absRoundPower < LONGWORD_DECIMAL_DIGITS) { + + // Lowest word gets integer rounding. + + // Clear rounding portion in lower longword and add 1 at right scale (roundMultiplyFactor). + + final long roundFactor = powerOfTenTable[absRoundPower]; + + final long r0 = + ((fast0 / roundFactor) * roundFactor) + + roundFactor; + result0 = r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fast1 + + r0 / MULTIPLER_LONGWORD_DECIMAL; + result1 = r1 % MULTIPLER_LONGWORD_DECIMAL; + result2 = + fast2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + + } else if (absRoundPower < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Middle word gets integer rounding; lower longword is cleared. + + final int adjustedAbsPower = absRoundPower - LONGWORD_DECIMAL_DIGITS; + + // Clear rounding portion in middle longword and add 1 at right scale (roundMultiplyFactor); + // lower longword result is 0; + + final long roundFactor = powerOfTenTable[adjustedAbsPower]; + + result0 = 0; + final long r1 = + ((fast1 / roundFactor) * roundFactor) + + roundFactor; + result1 = r1 % MULTIPLER_LONGWORD_DECIMAL; + result2 = + fast2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + + } else { + + // High word gets integer rounding; middle and lower longwords are cleared. + + final int adjustedAbsPower = absRoundPower - TWO_X_LONGWORD_DECIMAL_DIGITS; + + // Clear rounding portion in high longword and add 1 at right scale (roundMultiplyFactor); + // middle and lower longwords result is 0; + + final long roundFactor = powerOfTenTable[adjustedAbsPower]; + + result0 = 0; + result1 = 0; + result2 = + ((fast2 / roundFactor) * roundFactor) + + roundFactor; + + } + + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + } + + private static void doClearRoundIntegerPortion( + long fast0, long fast1, long fast2, + int absRoundPower, + FastHiveDecimal fastResult) { + + long result0; + long result1; + long result2; + + if (absRoundPower < LONGWORD_DECIMAL_DIGITS) { + + // Lowest word gets integer rounding. + + // Clear rounding portion in lower longword and add 1 at right scale (roundMultiplyFactor). + + final long roundFactor = powerOfTenTable[absRoundPower]; + // final long roundMultiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - absRoundPower]; + + result0 = + ((fast0 / roundFactor) * roundFactor); + result1 = fast1; + result2 = fast2; + + } else if (absRoundPower < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Middle word gets integer rounding; lower longword is cleared. + + final int adjustedAbsPower = absRoundPower - LONGWORD_DECIMAL_DIGITS; + + // Clear rounding portion in middle longword and add 1 at right scale (roundMultiplyFactor); + // lower longword result is 0; + + final long roundFactor = powerOfTenTable[adjustedAbsPower]; + + result0 = 0; + result1 = + ((fast1 / roundFactor) * roundFactor); + result2 = fast2; + + } else { + + // High word gets integer rounding; middle and lower longwords are cleared. + + final int adjustedAbsPower = absRoundPower - TWO_X_LONGWORD_DECIMAL_DIGITS; + + // Clear rounding portion in high longword and add 1 at right scale (roundMultiplyFactor); + // middle and lower longwords result is 0; + + final long roundFactor = powerOfTenTable[adjustedAbsPower]; + + result0 = 0; + result1 = 0; + result2 = + ((fast2 / roundFactor) * roundFactor); + + } + + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + } + + /** + * Fast decimal integer part rounding ROUND_UP. + * + * ceiling(12400.8302, -2) = 12500 // E.g. Positive case FAST_ROUND_CEILING + * rr rrrr + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param fastIntegerDigitCount the number of integer digits + * @param fastScale the scale of the number + * @param roundPower the power to round to + * @param fastResult an object to reuse + * @return was the operation successful + */ + public static boolean fastRoundIntegerUp( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int roundPower, + FastHiveDecimal fastResult) { + + /* + * Basic algorithm: + * + * 1. Determine if rounding part is non-zero for rounding. + * 2. Scale away fractional digits if present. + * 3. If rounding, clear integer rounding portion and add 1. + * + */ + + if (roundPower >= 0) { + throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")"); + } + + final int absRoundPower = -roundPower; + if (fastIntegerDigitCount < absRoundPower) { + + // Above decimal. + return false; + } + + final int roundingPoint = absRoundPower + fastScale; + if (roundingPoint > HiveDecimal.MAX_PRECISION) { + + // Value becomes null for rounding beyond. + return false; + } + + // First, determine whether rounding is necessary based on rounding point, which is inside + // integer part. And, get rid of any fractional digits. The result scale will be 0. + // + boolean isRoundPortionAllZeroes = + isRoundPortionAllZeroes( + fast0, fast1, fast2, + roundingPoint); + + // If necessary, divide and multiply to get rid of fractional digits. + if (fastScale == 0) { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } else { + doFastScaleDown( + fast0, fast1, fast2, + /* scaleDown */ fastScale, + fastResult); + } + + // The fractional digits are gone; when rounding, clear remaining round digits and add 1. + if (!isRoundPortionAllZeroes) { + + doClearRoundIntegerPortionAndAddOne( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + absRoundPower, + fastResult); + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = fastSignum; + fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult); + fastResult.fastScale = 0; + } + + return true; + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_DOWN. + * + * The fraction being scaled away is thrown away. + * + * The signum will be updated if the result is 0, otherwise the original sign is unchanged. + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param fastIntegerDigitCount the number of integer digits + * @param fastScale the scale of the number + * @param roundPower the power to round to + * @param fastResult an object to reuse + * @return was the operation successful? + */ + public static boolean fastRoundIntegerDown( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int roundPower, + FastHiveDecimal fastResult) { + + /* + * Basic algorithm: + * + * 1. Scale away fractional digits if present. + * 2. Clear integer rounding portion. + * + */ + + if (roundPower >= 0) { + throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")"); + } + + final int absRoundPower = -roundPower; + if (fastIntegerDigitCount < absRoundPower) { + + // Zero result. + fastResult.fastReset(); + return true; + } + + final int roundingPoint = absRoundPower + fastScale; + if (roundingPoint > HiveDecimal.MAX_PRECISION) { + + // Value becomes zero for rounding beyond. + fastResult.fastReset(); + return true; + } + + // If necessary, divide and multiply to get rid of fractional digits. + if (fastScale == 0) { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } else { + doFastScaleDown( + fast0, fast1, fast2, + /* scaleDown */ fastScale, + fastResult); + } + + // The fractional digits are gone; clear remaining round digits. + doClearRoundIntegerPortion( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + absRoundPower, + fastResult); + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = 0; + fastResult.fastSignum = fastSignum; + fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult); + fastResult.fastScale = 0; + } + + return true; + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP. + * + * When the fraction being scaled away is >= 0.5, the add 1. + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param fastIntegerDigitCount the number of integer digits + * @param fastScale the scale of the number + * @param roundPower the power to round to + * @param fastResult an object to reuse + * @return was the operation successful? + */ + public static boolean fastRoundIntegerHalfUp( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int roundPower, + FastHiveDecimal fastResult) { + + /* + * Basic algorithm: + * + * 1. Determine if rounding digit is >= 5 for rounding. + * 2. Scale away fractional digits if present. + * 3. If rounding, clear integer rounding portion and add 1. + * + */ + + if (roundPower >= 0) { + throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")"); + } + + final int absRoundPower = -roundPower; + if (fastIntegerDigitCount < absRoundPower) { + + // Zero result. + fastResult.fastReset(); + return true; + } + + final int roundingPoint = absRoundPower + fastScale; + if (roundingPoint > HiveDecimal.MAX_PRECISION) { + + // Value becomes zero for rounding beyond. + fastResult.fastReset(); + return true; + } + + // First, determine whether rounding is necessary based on rounding point, which is inside + // integer part. And, get rid of any fractional digits. The result scale will be 0. + // + boolean isRoundPortionHalfUp = + isRoundPortionHalfUp( + fast0, fast1, fast2, + roundingPoint); + + // If necessary, divide and multiply to get rid of fractional digits. + if (fastScale == 0) { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } else { + doFastScaleDown( + fast0, fast1, fast2, + /* scaleDown */ fastScale, + fastResult); + } + + // The fractional digits are gone; when rounding, clear remaining round digits and add 1. + if (isRoundPortionHalfUp) { + + doClearRoundIntegerPortionAndAddOne( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + absRoundPower, + fastResult); + } else { + + doClearRoundIntegerPortion( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + absRoundPower, + fastResult); + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = fastSignum; + fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult); + fastResult.fastScale = 0; + } + + return true; + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_EVEN. + * + * When the fraction being scaled away is exactly 0.5, then round and add 1 only if aaa. + * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1. + * Otherwise, throw away fraction. + * + * The signum will be updated if the result is 0, otherwise the original sign is unchanged. + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param fastIntegerDigitCount the number of integer digits + * @param fastScale the scale of the number + * @param roundPower the power to round to + * @param fastResult an object to reuse + * @return was the operation successful? + */ + public static boolean fastRoundIntegerHalfEven( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int roundPower, + FastHiveDecimal fastResult) { + + /* + * Basic algorithm: + * + * 1. Determine if rounding part meets banker's rounding rules for rounding. + * 2. Scale away fractional digits if present. + * 3. If rounding, clear integer rounding portion and add 1. + * + */ + + if (roundPower >= 0) { + throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")"); + } + + final int absRoundPower = -roundPower; + if (fastIntegerDigitCount < absRoundPower) { + + // Zero result. + fastResult.fastReset(); + } + + final int roundingPoint = absRoundPower + fastScale; + if (roundingPoint > HiveDecimal.MAX_PRECISION) { + + // Value becomes zero for rounding beyond. + fastResult.fastReset(); + return true; + } + + // First, determine whether rounding is necessary based on rounding point, which is inside + // integer part. And, get rid of any fractional digits. The result scale will be 0. + // + boolean isRoundPortionHalfEven = + isRoundPortionHalfEven( + fast0, fast1, fast2, + roundingPoint); + + // If necessary, divide and multiply to get rid of fractional digits. + if (fastScale == 0) { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } else { + doFastScaleDown( + fast0, fast1, fast2, + /* scaleDown */ fastScale, + fastResult); + } + + // The fractional digits are gone; when rounding, clear remaining round digits and add 1. + if (isRoundPortionHalfEven) { + + doClearRoundIntegerPortionAndAddOne( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + absRoundPower, + fastResult); + } else { + + doClearRoundIntegerPortion( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + absRoundPower, + fastResult); + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = fastSignum; + fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult); + fastResult.fastScale = 0; + } + + return true; + } + + /** + * Fast decimal scale down by factor of 10 and do not allow rounding. + * + * When the fraction being scaled away is non-zero, return false. + * + * The signum will be updated if the result is 0, otherwise the original sign + * is unchanged. + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param scaleDown the digits to scale down by + * @param fastResult an object to reuse + * @return was the operation successful? + */ + public static boolean fastScaleDownNoRound( + int fastSignum, long fast0, long fast1, long fast2, + int scaleDown, + FastHiveDecimal fastResult) { + if (scaleDown < 1 || scaleDown >= THREE_X_LONGWORD_DECIMAL_DIGITS - 1) { + throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 3*16 - 1 (scaleDown " + scaleDown + ")"); + } + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + if (scaleDown < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[scaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; + + final long throwAwayFraction = fast0 % divideFactor; + + if (throwAwayFraction != 0) { + return false; + } + result0 = + fast0 / divideFactor + + ((fast1 % divideFactor) * multiplyFactor); + result1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result2 = + fast2 / divideFactor; + + } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + boolean isThrowAwayFractionZero; + if (adjustedScaleDown == 0) { + isThrowAwayFractionZero = (fast0 == 0); + } else { + final long throwAwayFraction = fast1 % divideFactor; + isThrowAwayFractionZero = (throwAwayFraction == 0 && fast0 == 0); + } + + if (!isThrowAwayFractionZero) { + return false; + } + result0 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result1 = + fast2 / divideFactor; + result2 = 0; + + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + + boolean isThrowAwayFractionZero; + if (adjustedScaleDown == 0) { + isThrowAwayFractionZero = (fast0 == 0 && fast1 == 0); + } else { + final long throwAwayFraction = fast2 % divideFactor; + isThrowAwayFractionZero = (throwAwayFraction == 0 && fast0 == 0 && fast1 == 0); + } + + if (!isThrowAwayFractionZero) { + return false; + } + result0 = + fast2 / divideFactor; + result1 = 0; + result2 = 0; + } + + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastReset(); + } else { + fastResult.fastSignum = fastSignum; + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + } + + return true; + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_UP. + * + * When the fraction being scaled away is non-zero, the add 1. + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param scaleDown the number of integer digits to scale + * @param fastResult an object to reuse + * @return was the operation successfule? + */ + public static boolean fastRoundFractionalUp( + int fastSignum, long fast0, long fast1, long fast2, + int scaleDown, + FastHiveDecimal fastResult) { + if (scaleDown < 1 || scaleDown > HiveDecimal.MAX_SCALE) { + throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < " + HiveDecimal.MAX_SCALE + " (scaleDown " + scaleDown + ")"); + } + + if (scaleDown == HiveDecimal.MAX_SCALE) { + + // Examine all digits being thrown away to determine if result is 0 or 1. + if (fast0 == 0 && fast1 == 0 && fast2 == 0) { + + // Zero result. + fastResult.fastReset(); + } else { + fastResult.fastSet(fastSignum, /* fast0 */ 1, 0, 0, /* fastIntegerDigitCount */ 1, 0); + } + return true; + } + + boolean isRoundPortionAllZeroes = + isRoundPortionAllZeroes( + fast0, fast1, fast2, + scaleDown); + + doFastScaleDown( + fast0, fast1, fast2, + scaleDown, + fastResult); + + if (!isRoundPortionAllZeroes) { + final long r0 = fastResult.fast0 + 1; + fastResult.fast0 = + r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fastResult.fast1 + + r0 / MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast2 = + fastResult.fast2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = fastSignum; + } + + return (fastResult.fast2 <= MAX_HIGHWORD_DECIMAL); + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_DOWN. + * + * The fraction being scaled away is thrown away. + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param scaleDown the number of integer digits to scale + * @param fastResult an object to reuse + */ + public static void fastRoundFractionalDown( + int fastSignum, long fast0, long fast1, long fast2, + int scaleDown, + FastHiveDecimal fastResult) { + if (scaleDown < 1 || scaleDown > HiveDecimal.MAX_SCALE) { + throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 38 (scaleDown " + scaleDown + ")"); + } + + if (scaleDown == HiveDecimal.MAX_SCALE) { + + // Complete fractional digits shear off. Zero result. + fastResult.fastReset(); + return; + } + + doFastScaleDown( + fast0, fast1, fast2, + scaleDown, + fastResult); + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = fastSignum; + } + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP. + * + * When the fraction being scaled away is >= 0.5, the add 1. + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param scaleDown the number of integer digits to scale + * @param fastResult an object to reuse + * @return was the operation successfule? + */ + public static boolean fastRoundFractionalHalfUp( + int fastSignum, long fast0, long fast1, long fast2, + int scaleDown, + FastHiveDecimal fastResult) { + if (fastSignum == 0) { + throw new IllegalArgumentException("Unexpected zero value"); + } + if (scaleDown < 1 || scaleDown > HiveDecimal.MAX_SCALE) { + throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 38 (scaleDown " + scaleDown + ")"); + } + + if (scaleDown == HiveDecimal.MAX_SCALE) { + + // Check highest digit for rounding. + final long roundDigit = fast2 / powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1]; + if (roundDigit < 5) { + + // Zero result. + fastResult.fastReset(); + } else { + fastResult.fastSet(fastSignum, /* fast0 */ 1, 0, 0, /* fastIntegerDigitCount */ 1, 0); + } + return true; + } + + boolean isRoundPortionHalfUp = + isRoundPortionHalfUp( + fast0, fast1, fast2, + scaleDown); + + doFastScaleDown( + fast0, fast1, fast2, + scaleDown, + fastResult); + + if (isRoundPortionHalfUp) { + final long r0 = fastResult.fast0 + 1; + fastResult.fast0 = + r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fastResult.fast1 + + r0 / MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast2 = + fastResult.fast2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = fastSignum; + } + + return (fastResult.fast2 <= MAX_HIGHWORD_DECIMAL); + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP. + * + * When the fraction being scaled away is >= 0.5, the add 1. + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param fast3 word 3 + * @param fast4 word 4 + * @param scaleDown the number of integer digits to scale + * @param fastResult an object to reuse + * @return was the operation successfule? + */ + public static boolean fastRoundFractionalHalfUp5Words( + int fastSignum, long fast0, long fast1, long fast2, long fast3, long fast4, + int scaleDown, + FastHiveDecimal fastResult) { + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + long result3; + long result4; + if (scaleDown < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + // Divide down just before scaleDown to get round digit. + final long withRoundDigit = fast0 / powerOfTenTable[scaleDown - 1]; + final long roundDigit = withRoundDigit % 10; + + final long divideFactor = powerOfTenTable[scaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; + + if (roundDigit < 5) { + result0 = + withRoundDigit / 10 + + ((fast1 % divideFactor) * multiplyFactor); + result1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result2 = + + fast2 / divideFactor + + ((fast3 % divideFactor) * multiplyFactor); + result3 = + fast3 / divideFactor + + ((fast4 % divideFactor) * multiplyFactor); + result4 = + fast4 / divideFactor; + } else { + // Add rounding and handle carry. + final long r0 = + withRoundDigit / 10 + + ((fast1 % divideFactor) * multiplyFactor) + + 1; + result0 = r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor) + + r0 / MULTIPLER_LONGWORD_DECIMAL; + result1 = r1 % MULTIPLER_LONGWORD_DECIMAL; + final long r2 = + fast2 / divideFactor + + + ((fast3 % divideFactor) * multiplyFactor) + + r1 / MULTIPLER_LONGWORD_DECIMAL; + result2 = r2 % MULTIPLER_LONGWORD_DECIMAL; + final long r3 = + fast3 / divideFactor + + ((fast4 % divideFactor) * multiplyFactor) + + r2 / MULTIPLER_LONGWORD_DECIMAL; + result3 = r3 % MULTIPLER_LONGWORD_DECIMAL; + result4 = + fast4 / divideFactor + + r3 % MULTIPLER_LONGWORD_DECIMAL; + } + } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS; + + long roundDigit; + long fast1Scaled; + if (adjustedScaleDown == 0) { + // Grab round digit from lowest word. + roundDigit = fast0 / (MULTIPLER_LONGWORD_DECIMAL / 10); + fast1Scaled = fast1; + } else { + // Divide down just before scaleDown to get round digit. + final long withRoundDigit = fast1 / powerOfTenTable[adjustedScaleDown - 1]; + roundDigit = withRoundDigit % 10; + fast1Scaled = withRoundDigit / 10; + } + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + if (roundDigit < 5) { + result0 = + fast1Scaled + + ((fast2 % divideFactor) * multiplyFactor); + result1 = + fast2 / divideFactor + + ((fast3 % divideFactor) * multiplyFactor); + result2 = + fast3 / divideFactor + + ((fast4 % divideFactor) * multiplyFactor); + result3 = + fast4 / divideFactor; + } else { + // Add rounding and handle carry. + final long r0 = + fast1Scaled + + ((fast2 % divideFactor) * multiplyFactor) + + 1; + result0 = r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fast2 / divideFactor + + ((fast3 % divideFactor) * multiplyFactor) + + r0 / MULTIPLER_LONGWORD_DECIMAL; + result1 = r1 % MULTIPLER_LONGWORD_DECIMAL; + final long r2 = + fast3 / divideFactor + + ((fast4 % divideFactor) * multiplyFactor) + + r1 / MULTIPLER_LONGWORD_DECIMAL; + result2 = r2 % MULTIPLER_LONGWORD_DECIMAL; + result3 = + fast4 / divideFactor + + r2 / MULTIPLER_LONGWORD_DECIMAL; + } + result4 = 0; + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS; + + long roundDigit; + long fast2Scaled; + if (adjustedScaleDown == 0) { + // Grab round digit from middle word. + roundDigit = fast1 / (MULTIPLER_LONGWORD_DECIMAL / 10); + fast2Scaled = fast2; + } else { + // Divide down just before scaleDown to get round digit. + final long withRoundDigit = fast2 / powerOfTenTable[adjustedScaleDown - 1]; + roundDigit = withRoundDigit % 10; + fast2Scaled = withRoundDigit / 10; + } + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + if (roundDigit < 5) { + result0 = + fast2Scaled + + ((fast3 % divideFactor) * multiplyFactor); + result1 = + fast3 / divideFactor + + ((fast4 % divideFactor) * multiplyFactor); + result2 = + fast4 / divideFactor; + } else { + // Add rounding. + final long r0 = + fast2Scaled + + ((fast3 % divideFactor) * multiplyFactor) + + 1; + result0 = r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fast3 / divideFactor + + ((fast4 % divideFactor) * multiplyFactor) + + r0 / MULTIPLER_LONGWORD_DECIMAL; + result1 = r1 % MULTIPLER_LONGWORD_DECIMAL; + result2 = + fast4 / divideFactor + + r1 / MULTIPLER_LONGWORD_DECIMAL; + } + result3 = 0; + result4 = 0; + } + + if (result4 != 0 || result3 != 0) { + throw new RuntimeException("Unexpected overflow into result3 or result4"); + } + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastReset(); + } + fastResult.fastSignum = fastSignum; + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + + return (result2 <= MAX_HIGHWORD_DECIMAL); + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_EVEN. + * + * When the fraction being scaled away is exactly 0.5, then round and add 1 only if aaa. + * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1. + * Otherwise, throw away fraction. + * + * @param fastSignum the sign (-1, 0, or +1) + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param scaleDown the number of integer digits to scale + * @param fastResult an object to reuse + * @return was the operation successfule? + */ + public static boolean fastRoundFractionalHalfEven( + int fastSignum, long fast0, long fast1, long fast2, + int scaleDown, + FastHiveDecimal fastResult) { + if (scaleDown < 1 || scaleDown > HiveDecimal.MAX_SCALE) { + throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 38 (scaleDown " + scaleDown + ")"); + } + + if (scaleDown == HiveDecimal.MAX_SCALE) { + + // Check for rounding. + final long roundDivisor = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1]; + final long withRoundDigit = fast2 / roundDivisor; + final long roundDigit = withRoundDigit % 10; + final long fast2Scaled = withRoundDigit / 10; + boolean shouldRound; + if (roundDigit > 5) { + shouldRound = true; + } else if (roundDigit == 5) { + boolean exactlyOneHalf = (fast2Scaled == 0 && fast1 == 0 && fast0 == 0); + if (exactlyOneHalf) { + // Round to even 0. + shouldRound = false; + } else { + shouldRound = true; + } + } else { + shouldRound = false; + } + if (!shouldRound) { + + // Zero result. + fastResult.fastReset(); + } else { + fastResult.fastSet(fastSignum, /* fast0 */ 1, 0, 0, /* fastIntegerDigitCount */ 1, 0); + } + return true; + } + + boolean isRoundPortionHalfEven = + isRoundPortionHalfEven( + fast0, fast1, fast2, + scaleDown); + + doFastScaleDown( + fast0, fast1, fast2, + scaleDown, + fastResult); + + if (isRoundPortionHalfEven) { + final long r0 = fastResult.fast0 + 1; + fastResult.fast0 = + r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fastResult.fast1 + + r0 / MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast2 = + fastResult.fast2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = fastSignum; + } + + return (fastResult.fast2 <= MAX_HIGHWORD_DECIMAL); + } + + public static void doFastScaleDown( + FastHiveDecimal fastDec, + int scaleDown, + FastHiveDecimal fastResult) { + doFastScaleDown( + fastDec.fast0, fastDec.fast1, fastDec.fast2, + scaleDown, + fastResult); + } + + //************************************************************************************************ + // Decimal Scale Up/Down. + + /** + * Fast decimal scale down by factor of 10 with NO rounding. + * + * The signum will be updated if the result is 0, otherwise the original sign is unchanged. + * + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param scaleDown the number of integer digits to scale + * @param fastResult an object to reuse + */ + public static void doFastScaleDown( + long fast0, long fast1, long fast2, + int scaleDown, + FastHiveDecimal fastResult) { + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + if (scaleDown < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[scaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; + + result0 = + fast0 / divideFactor + + ((fast1 % divideFactor) * multiplyFactor); + result1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result2 = + fast2 / divideFactor; + + } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + result0 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result1 = + fast2 / divideFactor; + result2 = 0; + + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS; + + result0 = + fast2 / powerOfTenTable[adjustedScaleDown]; + result1 = 0; + result2 = 0; + + } + + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastSignum = 0; + } + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + } + + public static boolean fastScaleUp( + FastHiveDecimal fastDec, + int scaleUp, + FastHiveDecimal fastResult) { + return + fastScaleUp( + fastDec.fast0, fastDec.fast1, fastDec.fast2, + scaleUp, + fastResult); + } + + /** + * Fast decimal scale up by factor of 10. + * @param fast0 word 0 of the internal representation + * @param fast1 word 1 + * @param fast2 word 2 + * @param scaleUp the number of integer digits to scale up by + * @param fastResult an object to reuse + * @return was the operation successfule? + */ + public static boolean fastScaleUp( + long fast0, long fast1, long fast2, + int scaleUp, + FastHiveDecimal fastResult) { + if (scaleUp < 1 || scaleUp >= HiveDecimal.MAX_SCALE) { + throw new IllegalArgumentException("Expecting scaleUp > 0 and scaleUp < 38"); + } + + long result0; + long result1; + long result2; + + // Each range checks for overflow first, then moves digits. + if (scaleUp < HIGHWORD_DECIMAL_DIGITS) { + // Need to check if there are overflow digits in the high word. + + final long overflowFactor = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - scaleUp]; + if (fast2 / overflowFactor != 0) { + return false; + } + + final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleUp]; + final long multiplyFactor = powerOfTenTable[scaleUp]; + + result2 = + fast2 * multiplyFactor + + fast1 / divideFactor; + result1 = + (fast1 % divideFactor) * multiplyFactor + + fast0 / divideFactor; + result0 = + (fast0 % divideFactor) * multiplyFactor; + } else if (scaleUp < HIGHWORD_DECIMAL_DIGITS + LONGWORD_DECIMAL_DIGITS) { + // High word must be zero. Check for overflow digits in middle word. + + if (fast2 != 0) { + return false; + } + + final int adjustedScaleUp = scaleUp - HIGHWORD_DECIMAL_DIGITS; + + final int middleDigits = LONGWORD_DECIMAL_DIGITS - adjustedScaleUp; + final long overflowFactor = powerOfTenTable[middleDigits]; + if (fast1 / overflowFactor != 0) { + return false; + } + + if (middleDigits < HIGHWORD_DECIMAL_DIGITS) { + // Must fill high word from both middle and lower longs. + + final int highWordMoreDigits = HIGHWORD_DECIMAL_DIGITS - middleDigits; + final long multiplyFactor = powerOfTenTable[highWordMoreDigits]; + + final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - highWordMoreDigits]; + + result2 = + fast1 * multiplyFactor + + fast0 / divideFactor; + result1 = + (fast0 % divideFactor) * multiplyFactor; + result0 = 0; + } else if (middleDigits == HIGHWORD_DECIMAL_DIGITS) { + // Fill high long from middle long, and middle long from lower long. + + result2 = fast1; + result1 = fast0; + result0 = 0; + } else { + // Fill high long from some of middle long. + + final int keepMiddleDigits = middleDigits - HIGHWORD_DECIMAL_DIGITS; + final long divideFactor = powerOfTenTable[keepMiddleDigits]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - keepMiddleDigits]; + + result2 = + fast1 / divideFactor; + result1 = + (fast1 % divideFactor) * multiplyFactor + + fast0 / divideFactor; + result0 = + (fast0 % divideFactor) * multiplyFactor; + } + } else { + // High and middle word must be zero. Check for overflow digits in lower word. + + if (fast2 != 0 || fast1 != 0) { + return false; + } + + final int adjustedScaleUp = scaleUp - HIGHWORD_DECIMAL_DIGITS - LONGWORD_DECIMAL_DIGITS; + + final int lowerDigits = LONGWORD_DECIMAL_DIGITS - adjustedScaleUp; + final long overflowFactor = powerOfTenTable[lowerDigits]; + if (fast0 / overflowFactor != 0) { + return false; + } + + if (lowerDigits < HIGHWORD_DECIMAL_DIGITS) { + // Must fill high word from both middle and lower longs. + + final int highWordMoreDigits = HIGHWORD_DECIMAL_DIGITS - lowerDigits; + final long multiplyFactor = powerOfTenTable[highWordMoreDigits]; + + final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - highWordMoreDigits]; + + result2 = + fast0 * multiplyFactor; + result1 = 0; + result0 = 0; + } else if (lowerDigits == HIGHWORD_DECIMAL_DIGITS) { + // Fill high long from lower long. + + result2 = fast0; + result1 = 0; + result0 = 0; + } else { + // Fill high long and middle from some of lower long. + + final int keepLowerDigits = lowerDigits - HIGHWORD_DECIMAL_DIGITS; + final long keepLowerDivideFactor = powerOfTenTable[keepLowerDigits]; + final long keepLowerMultiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - keepLowerDigits]; + + result2 = + fast0 / keepLowerDivideFactor; + result1 = + (fast0 % keepLowerDivideFactor) * keepLowerMultiplyFactor; + result0 = 0; + } + } + + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastSignum = 0; + } + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + + return true; + } + + //************************************************************************************************ + // Decimal Precision / Trailing Zeroes. + + public static int fastLongWordTrailingZeroCount( + long longWord) { + + if (longWord == 0) { + return LONGWORD_DECIMAL_DIGITS; + } + + long factor = 10; + for (int i = 0; i < LONGWORD_DECIMAL_DIGITS; i++) { + if (longWord % factor != 0) { + return i; + } + factor *= 10; + } + return 0; + } + + public static int fastHighWordTrailingZeroCount( + long longWord) { + + if (longWord == 0) { + return HIGHWORD_DECIMAL_DIGITS; + } + + long factor = 10; + for (int i = 0; i < HIGHWORD_DECIMAL_DIGITS; i++) { + if (longWord % factor != 0) { + return i; + } + factor *= 10; + } + return 0; + } + + public static int fastLongWordPrecision( + long longWord) { + + if (longWord == 0) { + return 0; + } + // Search like binary search to minimize comparisons. + if (longWord > 99999999L) { + if (longWord > 999999999999L) { + if (longWord > 99999999999999L) { + if (longWord > 999999999999999L) { + return 16; + } else { + return 15; + } + } else { + if (longWord > 9999999999999L) { + return 14; + } else { + return 13; + } + } + } else { + if (longWord > 9999999999L) { + if (longWord > 99999999999L) { + return 12; + } else { + return 11; + } + } else { + if (longWord > 999999999L) { + return 10; + } else { + return 9; + } + } + } + } else { + if (longWord > 9999L) { + if (longWord > 999999L) { + if (longWord > 9999999L) { + return 8; + } else { + return 7; + } + } else { + if (longWord > 99999L) { + return 6; + } else { + return 5; + } + } + } else { + if (longWord > 99L) { + if (longWord > 999L) { + return 4; + } else { + return 3; + } + } else { + if (longWord > 9L) { + return 2; + } else { + return 1; + } + } + } + } + } + + public static int fastHighWordPrecision( + long longWord) { + + if (longWord == 0) { + return 0; + } + // 6 highword digits. + if (longWord > 999L) { + if (longWord > 9999L) { + if (longWord > 99999L) { + return 6; + } else { + return 5; + } + } else { + return 4; + } + } else { + if (longWord > 99L) { + return 3; + } else { + if (longWord > 9L) { + return 2; + } else { + return 1; + } + } + } + } + + public static int fastSqlPrecision( + FastHiveDecimal fastDec) { + return + fastSqlPrecision( + fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2, + fastDec.fastIntegerDigitCount, fastDec.fastScale); + } + + public static int fastSqlPrecision( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastSignum == 0) { + return 1; + } + + final int rawPrecision = + fastRawPrecision(fastSignum, fast0, fast1, fast2); + + if (rawPrecision < fastScale) { + // This can happen for numbers less than 0.1 + // For 0.001234: rawPrecision=4, scale=6 + // In this case, we'll set the type to have the same precision as the scale. + return fastScale; + } + return rawPrecision; + } + + public static int fastRawPrecision( + FastHiveDecimal fastDec) { + return + fastRawPrecision( + fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2); + } + + public static int fastRawPrecision( + int fastSignum, long fast0, long fast1, long fast2) { + + if (fastSignum == 0) { + return 0; + } + + int precision; + + if (fast2 != 0) { + + // 6 highword digits. + precision = TWO_X_LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(fast2); + + } else if (fast1 != 0) { + + // Check fast1. + precision = LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fast1); + + + } else { + + // Check fast0. + precision = fastLongWordPrecision(fast0); + + } + + return precision; + } + + // Determine if all digits below a power is zero. + // The lowest digit is power = 0. + public static boolean isAllZeroesBelow( + int fastSignum, long fast0, long fast1, long fast2, + int power) { + if (power < 0 || power > HiveDecimal.MAX_SCALE) { + throw new IllegalArgumentException("Expecting power >= 0 and power <= 38"); + } + + if (fastSignum == 0) { + return true; + } + + if (power >= TWO_X_LONGWORD_DECIMAL_DIGITS) { + if (fast0 != 0 || fast1 != 0) { + return false; + } + final int adjustedPower = power - TWO_X_LONGWORD_DECIMAL_DIGITS; + if (adjustedPower == 0) { + return true; + } + long remainder = fast2 % powerOfTenTable[adjustedPower]; + return (remainder == 0); + } else if (power >= LONGWORD_DECIMAL_DIGITS) { + if (fast0 != 0) { + return false; + } + final int adjustedPower = power - LONGWORD_DECIMAL_DIGITS; + if (adjustedPower == 0) { + return true; + } + long remainder = fast1 % powerOfTenTable[adjustedPower]; + return (remainder == 0); + } else { + if (power == 0) { + return true; + } + long remainder = fast0 % powerOfTenTable[power]; + return (remainder == 0); + } + } + + public static boolean fastExceedsPrecision( + long fast0, long fast1, long fast2, + int precision) { + + if (precision <= 0) { + return true; + } else if (precision >= HiveDecimal.MAX_PRECISION) { + return false; + } + final int precisionLessOne = precision - 1; + + // 0 (lowest), 1 (middle), or 2 (high). + int wordNum = precisionLessOne / LONGWORD_DECIMAL_DIGITS; + + int digitInWord = precisionLessOne % LONGWORD_DECIMAL_DIGITS; + + final long overLimitInWord = powerOfTenTable[digitInWord + 1] - 1; + + if (wordNum == 0) { + if (digitInWord < LONGWORD_DECIMAL_DIGITS - 1) { + if (fast0 > overLimitInWord) { + return true; + } + } + return (fast1 != 0 || fast2 != 0); + } else if (wordNum == 1) { + if (digitInWord < LONGWORD_DECIMAL_DIGITS - 1) { + if (fast1 > overLimitInWord) { + return true; + } + } + return (fast2 != 0); + } else { + // We've eliminated the highest digit already with HiveDecimal.MAX_PRECISION check above. + return (fast2 > overLimitInWord); + } + } + + public static int fastTrailingDecimalZeroCount( + long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + if (fastScale < 0 || fastScale > HiveDecimal.MAX_SCALE) { + throw new IllegalArgumentException("Expecting scale >= 0 and scale <= 38"); + } + if (fastScale == 0) { + return 0; + } + + final int lowerLongwordDigits = Math.min(fastScale, LONGWORD_DECIMAL_DIGITS); + if (lowerLongwordDigits < LONGWORD_DECIMAL_DIGITS || fast0 != 0) { + long factor = 10; + for (int i = 0; i < lowerLongwordDigits; i++) { + if (fast0 % factor != 0) { + return i; + } + factor *= 10; + } + if (lowerLongwordDigits < LONGWORD_DECIMAL_DIGITS) { + return fastScale; + } + } + if (fastScale == LONGWORD_DECIMAL_DIGITS) { + return fastScale; + } + final int middleLongwordDigits = Math.min(fastScale - LONGWORD_DECIMAL_DIGITS, LONGWORD_DECIMAL_DIGITS); + if (middleLongwordDigits < LONGWORD_DECIMAL_DIGITS || fast1 != 0) { + long factor = 10; + for (int i = 0; i < middleLongwordDigits; i++) { + if (fast1 % factor != 0) { + return LONGWORD_DECIMAL_DIGITS + i; + } + factor *= 10; + } + if (middleLongwordDigits < LONGWORD_DECIMAL_DIGITS) { + return fastScale; + } + } + if (fastScale == TWO_X_LONGWORD_DECIMAL_DIGITS) { + return fastScale; + } + final int highLongwordDigits = fastScale - TWO_X_LONGWORD_DECIMAL_DIGITS; + if (highLongwordDigits < HIGHWORD_DECIMAL_DIGITS || fast2 != 0) { + long factor = 10; + for (int i = 0; i < highLongwordDigits; i++) { + if (fast2 % factor != 0) { + return TWO_X_LONGWORD_DECIMAL_DIGITS + i; + } + factor *= 10; + } + } + return fastScale; + } + + public static FastCheckPrecisionScaleStatus fastCheckPrecisionScale( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int maxPrecision, int maxScale) { + + if (fastSignum == 0) { + return FastCheckPrecisionScaleStatus.NO_CHANGE; + } + final int maxIntegerDigitCount = maxPrecision - maxScale; + if (fastIntegerDigitCount > maxIntegerDigitCount) { + return FastCheckPrecisionScaleStatus.OVERFLOW; + } + if (fastScale > maxScale) { + return FastCheckPrecisionScaleStatus.UPDATE_SCALE_DOWN; + } + return FastCheckPrecisionScaleStatus.NO_CHANGE; + } + + public static boolean fastUpdatePrecisionScale( + final int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int maxPrecision, int maxScale, FastCheckPrecisionScaleStatus status, + FastHiveDecimal fastResult) { + + switch (status) { + case UPDATE_SCALE_DOWN: + { + fastResult.fastSignum = fastSignum; + + // Throw away lower digits. + if (!fastRoundFractionalHalfUp( + fastSignum, fast0, fast1, fast2, + fastScale - maxScale, + fastResult)) { + return false; + } + + fastResult.fastScale = maxScale; + + // CONSIDER: For now, recompute integerDigitCount... + fastResult.fastIntegerDigitCount = + Math.max(0, fastRawPrecision(fastResult) - fastResult.fastScale); + + // And, round up may cause us to exceed our precision/scale... + final int maxIntegerDigitCount = maxPrecision - maxScale; + if (fastResult.fastIntegerDigitCount > maxIntegerDigitCount) { + return false; + } + + // Scaling down may have opened up trailing zeroes... + final int trailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (trailingZeroCount > 0) { + // Scale down again. + doFastScaleDown( + fastResult, + trailingZeroCount, + fastResult); + fastResult.fastScale -= trailingZeroCount; + } + } + break; + default: + throw new RuntimeException("Unexpected fast check precision scale status " + status); + } + + return true; + } + + //************************************************************************************************ + // Decimal Addition / Subtraction. + + public static boolean doAddSameScaleSameSign( + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + doAddSameScaleSameSign( + /* resultSignum */ fastLeft.fastSignum, + fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastResult); + } + + public static boolean doAddSameScaleSameSign( + int resultSignum, + long left0, long left1, long left2, + long right0, long right1, long right2, + FastHiveDecimal fastResult) { + + long result0; + long result1; + long result2; + + final long r0 = left0 + right0; + result0 = + r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + left1 + + right1 + + r0 / MULTIPLER_LONGWORD_DECIMAL; + result1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + result2 = + left2 + + right2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastReset(); + } else { + fastResult.fastSignum = resultSignum; + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + } + + return (result2 <= MAX_HIGHWORD_DECIMAL); + } + + public static boolean doSubtractSameScaleNoUnderflow( + int resultSignum, + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + doSubtractSameScaleNoUnderflow( + resultSignum, + fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastResult); + } + + public static boolean doSubtractSameScaleNoUnderflow( + int resultSignum, + long left0, long left1, long left2, + long right0, long right1, long right2, + FastHiveDecimal fastResult) { + + long result0; + long result1; + long result2; + + final long r0 = left0 - right0; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = left1 - right1 - 1; + } else { + result0 = r0; + r1 = left1 - right1; + } + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + result2 = left2 - right2 - 1; + } else { + result1 = r1; + result2 = left2 - right2; + } + if (result2 < 0) { + return false; + } + + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastReset(); + } else { + fastResult.fastSignum = resultSignum; + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + } + + return true; + } + + public static boolean doSubtractSameScaleNoUnderflow( + long left0, long left1, long left2, + long right0, long right1, long right2, + long[] result) { + + long result0; + long result1; + long result2; + + final long r0 = left0 - right0; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = left1 - right1 - 1; + } else { + result0 = r0; + r1 = left1 - right1; + } + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + result2 = left2 - right2 - 1; + } else { + result1 = r1; + result2 = left2 - right2; + } + if (result2 < 0) { + return false; + } + + result[0] = result0; + result[1] = result1; + result[2] = result2; + + return true; + } + + private static boolean doAddSameScale( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int scale, + FastHiveDecimal fastResult) { + + if (leftSignum == rightSignum) { + if (!doAddSameScaleSameSign( + /* resultSignum */ leftSignum, + leftFast0, leftFast1, leftFast2, + rightFast0, rightFast1, rightFast2, + fastResult)) { + // Handle overflow precision issue. + if (scale > 0) { + if (!fastRoundFractionalHalfUp( + fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, + 1, + fastResult)) { + return false; + } + scale--; + } else { + // Overflow. + return false; + } + } + fastResult.fastScale = scale; + } else { + // Just compare the magnitudes (i.e. signums set to 1). + int compareTo = + fastCompareTo( + 1, + leftFast0, leftFast1, leftFast2, 0, + 1, + rightFast0, rightFast1, rightFast2, 0); + if (compareTo == 0) { + // They cancel each other. + fastResult.fastReset(); + return true; + } + + if (compareTo == 1) { + if (!doSubtractSameScaleNoUnderflow( + /* resultSignum */ leftSignum, + leftFast0, leftFast1, leftFast2, + rightFast0, rightFast1, rightFast2, + fastResult)) { + throw new RuntimeException("Unexpected underflow"); + } + } else { + if (!doSubtractSameScaleNoUnderflow( + /* resultSignum */ rightSignum, + rightFast0, rightFast1, rightFast2, + leftFast0, leftFast1, leftFast2, + fastResult)) { + throw new RuntimeException("Unexpected underflow"); + } + } + fastResult.fastScale = scale; + } + + if (fastResult.fastSignum != 0) { + final int precision = fastRawPrecision(fastResult); + fastResult.fastIntegerDigitCount = Math.max(0, precision - fastResult.fastScale); + } + + final int resultTrailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (resultTrailingZeroCount > 0) { + doFastScaleDown( + fastResult, + resultTrailingZeroCount, + fastResult); + if (fastResult.fastSignum == 0) { + fastResult.fastScale = 0; + } else { + fastResult.fastScale -= resultTrailingZeroCount; + } + } + + return true; + } + + /** + * Handle the common logic at the end of fastAddDifferentScale / fastSubtractDifferentScale that + * takes the 5 intermediate result words and fits them into the 3 longword fast result. + * + */ + private static boolean doFinishAddSubtractDifferentScale( + long result0, long result1, long result2, long result3, long result4, + int resultScale, + FastHiveDecimal fastResult) { + + int precision; + if (result4 != 0) { + precision = FOUR_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result4); + } else if (result3 != 0) { + precision = THREE_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result3); + } else if (result2 != 0) { + precision = TWO_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result2); + } else if (result1 != 0) { + precision = LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result1); + } else { + precision = fastLongWordPrecision(result0); + } + + if (precision > HiveDecimal.MAX_PRECISION){ + final int scaleDown = precision - HiveDecimal.MAX_PRECISION; + + resultScale -= scaleDown; + if (resultScale < 0) { + // No room. + return false; + } + if (!fastRoundFractionalHalfUp5Words( + 1, result0, result1, result2, result3, result4, + scaleDown, + fastResult)) { + // Handle overflow precision issue. + if (resultScale > 0) { + if (!fastRoundFractionalHalfUp( + fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, + 1, + fastResult)) { + throw new RuntimeException("Unexpected overflow"); + } + if (fastResult.fastSignum == 0) { + return true; + } + resultScale--; + } else { + return false; + } + } + + precision = fastRawPrecision(1, fastResult.fast0, fastResult.fast1, fastResult.fast2); + + // Stick back into result variables... + result0 = fastResult.fast0; + result1 = fastResult.fast1; + result2 = fastResult.fast2; + } + + // Caller will set signum. + fastResult.fastSignum = 1; + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + fastResult.fastIntegerDigitCount = Math.max(0, precision - resultScale); + fastResult.fastScale = resultScale; + + final int resultTrailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (resultTrailingZeroCount > 0) { + doFastScaleDown( + fastResult, + resultTrailingZeroCount, + fastResult); + if (fastResult.fastSignum == 0) { + fastResult.fastScale = 0; + } else { + fastResult.fastScale -= resultTrailingZeroCount; + } + } + + return true; + } + + /** + * Handle decimal subtraction when the values have different scales. + */ + private static boolean fastSubtractDifferentScale( + long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + int diffScale; + int resultScale; + + long result0 = 0; + long result1 = 0; + long result2 = 0; + long result3 = 0; + long result4 = 0; + + // Since subtraction is not commutative, we can must subtract in the order passed in. + if (leftScale > rightScale) { + + // Since left has a longer digit tail and it doesn't move; we will shift the right digits + // as we do our addition into the result. + + diffScale = leftScale - rightScale; + resultScale = leftScale; + + if (diffScale < LONGWORD_DECIMAL_DIGITS) { + + final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale]; + + final long r0 = + leftFast0 + - (rightFast0 % divideFactor) * multiplyFactor; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = + leftFast1 + - rightFast0 / divideFactor + - (rightFast1 % divideFactor) * multiplyFactor + - 1; + } else { + result0 = r0; + r1 = + leftFast1 + - rightFast0 / divideFactor + - (rightFast1 % divideFactor) * multiplyFactor; + } + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + leftFast2 + - rightFast1 / divideFactor + - (rightFast2 % divideFactor) * multiplyFactor + - 1; + } else { + result1 = r1; + r2 = + leftFast2 + - rightFast1 / divideFactor + - (rightFast2 % divideFactor) * multiplyFactor; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + -(rightFast2 / divideFactor) + - 1; + } else { + result2 = r2; + r3 = + -(rightFast2 / divideFactor); + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = - 1; + } else { + result3 = r3; + r4 = 0; + } + if (r4 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + + } else if (diffScale == LONGWORD_DECIMAL_DIGITS){ + + result0 = leftFast0; + final long r1 = + leftFast1 + - rightFast0; + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + leftFast2 + - rightFast1 + - 1; + } else { + result1 = r1; + r2 = + leftFast2 + - rightFast1; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + -rightFast2 + - 1; + } else { + result2 = r2; + r3 = + -rightFast2; + } + if (r3 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + + + } else if (diffScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + final long divideFactor = powerOfTenTable[TWO_X_LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale - LONGWORD_DECIMAL_DIGITS]; + + result0 = leftFast0; + final long r1 = + leftFast1 + -(rightFast0 % divideFactor) * multiplyFactor; + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + leftFast2 + - rightFast0 / divideFactor + - (rightFast1 % divideFactor) * multiplyFactor + - 1; + } else { + result1 = r1; + r2 = + leftFast2 + - rightFast0 / divideFactor + - (rightFast1 % divideFactor) * multiplyFactor; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + - rightFast1 / divideFactor + - (rightFast2 % divideFactor) * multiplyFactor + - 1; + } else { + result2 = r2; + r3 = + - rightFast1 / divideFactor + - (rightFast2 % divideFactor) * multiplyFactor; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = + - rightFast2 / divideFactor + - 1; + } else { + result3 = r3; + r4 = + - rightFast2 / divideFactor; + } + long r5; + if (r4 < 0) { + result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; + r5 = - 1; + } else { + result4 = r4; + r5 = 0; + } + if (r5 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + + } else if (diffScale == TWO_X_LONGWORD_DECIMAL_DIGITS) { + + result0 = leftFast0; + result1 = leftFast1; + final long r2 = + leftFast2 + - rightFast0; + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + - rightFast1 + - 1; + } else { + result2 = r2; + r3 = + - rightFast1; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = + -rightFast2 + - 1; + } else { + result3 = r3; + r4 = + -rightFast2; + } + long r5; + if (r4 < 0) { + result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; + r5 = - 1; + } else { + result4 = r4; + r5 = 0; + } + if (r5 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + + } else { + + final long divideFactor = powerOfTenTable[THREE_X_LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale - TWO_X_LONGWORD_DECIMAL_DIGITS]; + + result0 = leftFast0; + result1 = leftFast1; + final long r2 = + leftFast2 + - (rightFast0 % divideFactor) * multiplyFactor; + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + - (rightFast0 / divideFactor) + - (rightFast1 % divideFactor) * multiplyFactor + - 1; + } else { + result2 = r2; + r3 = + - (rightFast0 / divideFactor) + - (rightFast1 % divideFactor) * multiplyFactor; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = + - (rightFast1 / divideFactor) + - (rightFast2 % divideFactor) * multiplyFactor + - 1; + } else { + result3 = r3; + r4 = + - (rightFast1 / divideFactor) + - (rightFast2 % divideFactor) * multiplyFactor; + } + long r5; + if (r4 < 0) { + result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; + r5 = + - (rightFast2 / divideFactor) + - 1; + } else { + result4 = r4; + r5 = + - (rightFast2 / divideFactor); + } + if (r5 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + } + } else { + + // Since right has a longer digit tail and it doesn't move; we will shift the left digits + // as we do our addition into the result. + + diffScale = rightScale - leftScale; + resultScale = rightScale; + + if (diffScale < LONGWORD_DECIMAL_DIGITS) { + + final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale]; + + final long r0 = + (leftFast0 % divideFactor) * multiplyFactor + - rightFast0; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = + leftFast0 / divideFactor + + (leftFast1 % divideFactor) * multiplyFactor + - rightFast1 + - 1; + } else { + result0 = r0; + r1 = + leftFast0 / divideFactor + + (leftFast1 % divideFactor) * multiplyFactor + - rightFast1; + } + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + leftFast1 / divideFactor + + (leftFast2 % divideFactor) * multiplyFactor + - rightFast2 + - 1; + } else { + result1 = r1; + r2 = + leftFast1 / divideFactor + + (leftFast2 % divideFactor) * multiplyFactor + - rightFast2; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + leftFast2 / divideFactor + - 1; + } else { + result2 = r2; + r3 = + leftFast2 / divideFactor; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = - 1; + } else { + result3 = r3; + r4 = 0; + } + if (r4 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + + } else if (diffScale == LONGWORD_DECIMAL_DIGITS){ + + final long r0 = + - rightFast0; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = + leftFast0 + - rightFast1 + - 1; + } else { + result0 = r0; + r1 = + leftFast0 + - rightFast1; + } + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + leftFast1 + - rightFast2 + - 1; + } else { + result1 = r1; + r2 = + leftFast1 + - rightFast2; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + leftFast2 + - 1; + } else { + result2 = r2; + r3 = + leftFast2; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = - 1; + } else { + result3 = r3; + r4 = 0; + } + if (r4 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + + } else if (diffScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + final long divideFactor = powerOfTenTable[TWO_X_LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale - LONGWORD_DECIMAL_DIGITS]; + + final long r0 = + - rightFast0; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = + (leftFast0 % divideFactor) * multiplyFactor + - rightFast1 + - 1; + } else { + result0 = r0; + r1 = + (leftFast0 % divideFactor) * multiplyFactor + - rightFast1; + } + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + leftFast0 / divideFactor + + (leftFast1 % divideFactor) * multiplyFactor + - rightFast2 + - 1; + } else { + result1 = r1; + r2 = + leftFast0 / divideFactor + + (leftFast1 % divideFactor) * multiplyFactor + - rightFast2; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + leftFast1 / divideFactor + + (leftFast2 % divideFactor) * multiplyFactor + - 1; + } else { + result2 = r2; + r3 = + leftFast1 / divideFactor + + (leftFast2 % divideFactor) * multiplyFactor; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = + leftFast2 / divideFactor + - 1; + } else { + result3 = r3; + r4 = + leftFast2 / divideFactor; + } + if (r4 < 0) { + result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; + } else { + result4 = r4; + } + + } else if (diffScale == TWO_X_LONGWORD_DECIMAL_DIGITS) { + + final long r0 = + - rightFast0; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = + - rightFast1 + - 1; + } else { + result0 = r0; + r1 = + - rightFast1; + } + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + leftFast0 + - rightFast2 + - 1; + } else { + result1 = r1; + r2 = + leftFast0 + - rightFast2; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + leftFast1 + - 1; + } else { + result2 = r2; + r3 = + leftFast1; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = + leftFast2 + - 1; + } else { + result3 = r3; + r4 = + leftFast2; + } + long r5; + if (r4 < 0) { + result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; + r5 = - 1; + } else { + result4 = r4; + r5 = 0; + } + if (r5 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + + } else { + + final long divideFactor = powerOfTenTable[THREE_X_LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale - TWO_X_LONGWORD_DECIMAL_DIGITS]; + + final long r0 = + - rightFast0; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = + - rightFast1 + - 1; + } else { + result0 = r0; + r1 = + - rightFast1; + } + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + (leftFast0 % divideFactor) * multiplyFactor + - rightFast2 + - 1; + } else { + result1 = r1; + r2 = + (leftFast0 % divideFactor) * multiplyFactor + - rightFast2; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + leftFast0 / divideFactor + + (leftFast1 % divideFactor) * multiplyFactor + - 1; + } else { + result2 = r2; + r3 = + leftFast0 / divideFactor + + (leftFast1 % divideFactor) * multiplyFactor; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = + leftFast1 / divideFactor + + (leftFast2 % divideFactor) * multiplyFactor + - 1; + } else { + result3 = r3; + r4 = + leftFast1 / divideFactor + + (leftFast2 % divideFactor) * multiplyFactor; + } + long r5; + if (r4 < 0) { + result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; + r5 = + leftFast2 / divideFactor + - 1; + } else { + result4 = r4; + r5 = + leftFast2 / divideFactor; + } + if (r5 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + } + } + + return + doFinishAddSubtractDifferentScale( + result0, result1, result2, result3, result4, + resultScale, + fastResult); + } + + /** + * Handle decimal addition when the values have different scales. + */ + private static boolean fastAddDifferentScale( + long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + // Arrange so result* has a longer digit tail and it lines up; we will shift the shift* digits + // as we do our addition and them into the result. + long result0; + long result1; + long result2; + + long shift0; + long shift1; + long shift2; + + int diffScale; + int resultScale; + + // Since addition is commutative, we can add in any order. + if (leftScale > rightScale) { + + result0 = leftFast0; + result1 = leftFast1; + result2 = leftFast2; + + shift0 = rightFast0; + shift1 = rightFast1; + shift2 = rightFast2; + + diffScale = leftScale - rightScale; + resultScale = leftScale; + } else { + + result0 = rightFast0; + result1 = rightFast1; + result2 = rightFast2; + + shift0 = leftFast0; + shift1 = leftFast1; + shift2 = leftFast2; + + diffScale = rightScale - leftScale; + resultScale = rightScale; + } + + long result3 = 0; + long result4 = 0; + + if (diffScale < LONGWORD_DECIMAL_DIGITS) { + + final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale]; + + final long r0 = + result0 + + (shift0 % divideFactor) * multiplyFactor; + result0 = + r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + result1 + + shift0 / divideFactor + + (shift1 % divideFactor) * multiplyFactor + + r0 / MULTIPLER_LONGWORD_DECIMAL; + result1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + final long r2 = + result2 + + shift1 / divideFactor + + (shift2 % divideFactor) * multiplyFactor + + r1 / MULTIPLER_LONGWORD_DECIMAL; + result2 = + r2 % MULTIPLER_LONGWORD_DECIMAL; + final long r3 = + shift2 / divideFactor + + r2 / MULTIPLER_LONGWORD_DECIMAL; + result3 = + r3 % MULTIPLER_LONGWORD_DECIMAL; + + } else if (diffScale == LONGWORD_DECIMAL_DIGITS){ + + final long r1 = + result1 + + shift0; + result1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + final long r2 = + result2 + + shift1 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + result2 = + r2 % MULTIPLER_LONGWORD_DECIMAL; + final long r3 = + shift2 + + r2 / MULTIPLER_LONGWORD_DECIMAL; + result3 = + r3 % MULTIPLER_LONGWORD_DECIMAL; + result4 = r3 / MULTIPLER_LONGWORD_DECIMAL; + + } else if (diffScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + final long divideFactor = powerOfTenTable[TWO_X_LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale - LONGWORD_DECIMAL_DIGITS]; + + final long r1 = + result1 + + (shift0 % divideFactor) * multiplyFactor; + result1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + final long r2 = + result2 + + shift0 / divideFactor + + (shift1 % divideFactor) * multiplyFactor + + r1 / MULTIPLER_LONGWORD_DECIMAL; + result2 = + r2 % MULTIPLER_LONGWORD_DECIMAL; + final long r3 = + shift1 / divideFactor + + (shift2 % divideFactor) * multiplyFactor + + r2 / MULTIPLER_LONGWORD_DECIMAL; + result3 = + r3 % MULTIPLER_LONGWORD_DECIMAL; + final long r4 = + shift2 / divideFactor + + r3 / MULTIPLER_LONGWORD_DECIMAL; + result4 = + r4 % MULTIPLER_LONGWORD_DECIMAL; + + } else if (diffScale == TWO_X_LONGWORD_DECIMAL_DIGITS) { + + final long r2 = + result2 + + shift0; + result2 = + r2 % MULTIPLER_LONGWORD_DECIMAL; + final long r3 = + shift1 + + r2 / MULTIPLER_LONGWORD_DECIMAL; + result3 = + r3 % MULTIPLER_LONGWORD_DECIMAL; + final long r4 = + shift2 + + r3 / MULTIPLER_LONGWORD_DECIMAL; + result4 = + r4 % MULTIPLER_LONGWORD_DECIMAL; + + } else { + + final long divideFactor = powerOfTenTable[THREE_X_LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale - TWO_X_LONGWORD_DECIMAL_DIGITS]; + + final long r2 = + result2 + + (shift0 % divideFactor) * multiplyFactor; + result2 = + r2 % MULTIPLER_LONGWORD_DECIMAL; + final long r3 = + shift0 / divideFactor + + (shift1 % divideFactor) * multiplyFactor + + r2 / MULTIPLER_LONGWORD_DECIMAL; + result3 = + r3 % MULTIPLER_LONGWORD_DECIMAL; + final long r4 = + shift1 / divideFactor + + (shift2 % divideFactor) * multiplyFactor + + r3 / MULTIPLER_LONGWORD_DECIMAL; + result4 = + r4 % MULTIPLER_LONGWORD_DECIMAL; + if (shift2 / divideFactor != 0) { + throw new RuntimeException("Unexpected overflow"); + } + + } + + return + doFinishAddSubtractDifferentScale( + result0, result1, result2, result3, result4, + resultScale, + fastResult); + } + + private static boolean doAddDifferentScale( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + if (leftSignum == rightSignum) { + if (!fastAddDifferentScale( + leftFast0, leftFast1, leftFast2, + leftIntegerDigitCount, leftScale, + rightFast0, rightFast1, rightFast2, + rightIntegerDigitCount, rightScale, + fastResult)) { + return false; + } + // Sign stays the same. + fastResult.fastSignum = leftSignum; + } else { + + // Just compare the magnitudes (i.e. signums set to 1). + int compareTo = + fastCompareTo( + 1, + leftFast0, leftFast1, leftFast2, leftScale, + 1, + rightFast0, rightFast1, rightFast2, rightScale); + if (compareTo == 0) { + // They cancel each other. + fastResult.fastSignum = 0; + fastResult.fast0 = 0; + fastResult.fast1 = 0; + fastResult.fast2 = 0; + fastResult.fastScale = 0; + return true; + } + + if (compareTo == 1) { + if (!fastSubtractDifferentScale( + leftFast0, leftFast1, leftFast2, + leftIntegerDigitCount, leftScale, + rightFast0, rightFast1, rightFast2, + rightIntegerDigitCount, rightScale, + fastResult)) { + throw new RuntimeException("Unexpected overflow"); + } + fastResult.fastSignum = leftSignum; + } else { + if (!fastSubtractDifferentScale( + rightFast0, rightFast1, rightFast2, + rightIntegerDigitCount, rightScale, + leftFast0, leftFast1, leftFast2, + leftIntegerDigitCount, leftScale, + fastResult)) { + throw new RuntimeException("Unexpected overflow"); + } + fastResult.fastSignum = rightSignum; + } + } + + final int resultTrailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (resultTrailingZeroCount > 0) { + doFastScaleDown( + fastResult, + resultTrailingZeroCount, + fastResult); + if (fastResult.fastSignum == 0) { + fastResult.fastScale = 0; + } else { + fastResult.fastScale -= resultTrailingZeroCount; + } + } + + return true; + } + + public static boolean fastAdd( + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return fastAdd( + fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastLeft.fastIntegerDigitCount, fastLeft.fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + /** + * Add the two decimals. + * + * NOTE: Scale Determination for Addition/Subtraction + * + * One could take the Math.min of the scales and adjust the operand with the lower scale have a + * scale = higher scale. + * + * But this does not seem to work with decimals with widely varying scales as these: + * + * 598575157855521918987423259.94094 dec1 (int digits 27,scale 5) + * + 0.0000000000006711991169422033 dec2 (int digits 0, scale 28) + * + * Trying to make dec1 to have a scale of 28 (i.e. by adding trailing zeroes) would exceed + * MAX_PRECISION (int digits 27 + 28 > 38). + * + * In this example we need to make sure we have enough integer digit room in the result to + * handle dec1's digits. In order to maintain that, we will need to get rid of lower + * fractional digits of dec2. But when do we do that? + * + * OldHiveDecimal.add does the full arithmetic add with all the digits using BigDecimal and + * then adjusts the result to fit in MAX_PRECISION, etc. + * + * If we try to do pre-rounding dec2 it is problematic. We'd need to know if there is a carry in + * the arithmetic in order to know at which scale to do the rounding. This gets complicated. + * + * So, the simplest thing is to emulate what OldHiveDecimal does and do the full digit addition + * and then fit the result afterwards. + * + * @param leftSignum The left sign (-1, 0, or +1) + * @param leftFast0 The left word 0 of reprentation + * @param leftFast1 word 1 + * @param leftFast2 word 2 + * @param leftIntegerDigitCount The left number of integer digits + * @param leftScale the left scale + * @param rightSignum The right sign (-1, 0, or +1) + * @param rightFast0 The right word 0 of reprentation + * @param rightFast1 word 1 + * @param rightFast2 word 2 + * @param rightIntegerDigitCount The right number of integer digits + * @param rightScale the right scale + * @param fastResult an object to reuse + * @return True if the addition was successful; Otherwise, false is returned on overflow. + */ + public static boolean fastAdd( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + if (rightSignum == 0) { + fastResult.fastSet(leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale); + return true; + } + if (leftSignum == 0) { + fastResult.fastSet(rightSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale); + return true; + } + + if (leftScale == rightScale) { + return doAddSameScale( + leftSignum, leftFast0, leftFast1, leftFast2, + rightSignum, rightFast0, rightFast1, rightFast2, + leftScale, + fastResult); + } else { + return doAddDifferentScale( + leftSignum, leftFast0, leftFast1, leftFast2, + leftIntegerDigitCount, leftScale, + rightSignum, rightFast0, rightFast1, rightFast2, + rightIntegerDigitCount, rightScale, + fastResult); + } + } + + public static boolean fastSubtract( + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return fastSubtract( + fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastLeft.fastIntegerDigitCount, fastLeft.fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + public static boolean fastSubtract( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + if (rightSignum == 0) { + fastResult.fastSet(leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale); + return true; + } + final int flippedDecSignum = (rightSignum == 1 ? -1 : 1); + if (leftSignum == 0) { + fastResult.fastSet(flippedDecSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale); + return true; + } + + if (leftScale == rightScale) { + return doAddSameScale( + leftSignum, leftFast0, leftFast1, leftFast2, + flippedDecSignum, rightFast0, rightFast1, rightFast2, + leftScale, + fastResult); + } else { + return doAddDifferentScale( + leftSignum, leftFast0, leftFast1, leftFast2, + leftIntegerDigitCount, leftScale, + flippedDecSignum, rightFast0, rightFast1, rightFast2, + rightIntegerDigitCount, rightScale, + fastResult); + } + } + + //************************************************************************************************ + // Decimal Multiply. + + private static boolean doMultiply( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + // Set signum before; if result is zero, fastMultiply will set signum to 0. + fastResult.fastSignum = (leftSignum == rightSignum ? 1 : -1); + int resultScale = leftScale + rightScale; + + /* + * For multiplicands with scale 0, trim trailing zeroes. + */ + if (leftScale == 0) { + + // Pretend like it has fractional digits so we can get the trailing zero count. + final int leftTrailingZeroCount = + fastTrailingDecimalZeroCount( + leftFast0, leftFast1, leftFast2, + 0, leftIntegerDigitCount); + if (leftTrailingZeroCount > 0) { + doFastScaleDown( + leftFast0, leftFast1, leftFast2, leftTrailingZeroCount, fastResult); + resultScale -= leftTrailingZeroCount; + leftFast0 = fastResult.fast0; + leftFast1 = fastResult.fast1; + leftFast2 = fastResult.fast2; + } + } + if (rightScale == 0) { + + // Pretend like it has fractional digits so we can get the trailing zero count. + final int rightTrailingZeroCount = + fastTrailingDecimalZeroCount( + rightFast0, rightFast1, rightFast2, + 0, rightIntegerDigitCount); + if (rightTrailingZeroCount > 0) { + doFastScaleDown( + rightFast0, rightFast1, rightFast2, rightTrailingZeroCount, fastResult); + resultScale -= rightTrailingZeroCount; + rightFast0 = fastResult.fast0; + rightFast1 = fastResult.fast1; + rightFast2 = fastResult.fast2; + } + } + + boolean largeOverflow = + !fastMultiply5x5HalfWords( + leftFast0, leftFast1, leftFast2, + rightFast0, rightFast1, rightFast2, + fastResult); + if (largeOverflow) { + return false; + } + + if (fastResult.fastSignum == 0) { + fastResult.fastScale = 0; + return true; + } + + if (resultScale < 0) { + if (-resultScale >= HiveDecimal.MAX_SCALE) { + return false; + } + if (!fastScaleUp( + fastResult.fast0, fastResult.fast1, fastResult.fast2, -resultScale, + fastResult)) { + return false; + } + resultScale = 0; + } + + int precision; + if (fastResult.fast2 != 0) { + precision = TWO_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast2); + } else if (fastResult.fast1 != 0) { + precision = LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast1); + } else { + precision = fastLongWordPrecision(fastResult.fast0); + } + + int integerDigitCount = Math.max(0, precision - resultScale); + if (integerDigitCount > HiveDecimal.MAX_PRECISION) { + // Integer is too large -- cannot recover by trimming fractional digits. + return false; + } + + if (precision > HiveDecimal.MAX_PRECISION || resultScale > HiveDecimal.MAX_SCALE) { + + // Trim off lower fractional digits but with NO ROUNDING. + + final int maxScale = HiveDecimal.MAX_SCALE - integerDigitCount; + final int scaleDown = resultScale - maxScale; + if (!fastScaleDownNoRound( + fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, + scaleDown, + fastResult)) { + // Round fractional must be 0. Not allowed to throw away digits. + return false; + } + resultScale -= scaleDown; + } + fastResult.fastScale = resultScale; + + // This assume no round up... + fastResult.fastIntegerDigitCount = integerDigitCount; + + if (fastResult.fastScale > HiveDecimal.MAX_SCALE) { + // We are not allowed to lose digits in multiply to be compatible with OldHiveDecimal + // behavior, so overflow. + // CONSIDER: Does it make sense to be so restrictive. If we just did repeated addition, + // it would succeed... + return false; + } + final int resultTrailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (resultTrailingZeroCount > 0) { + doFastScaleDown( + fastResult, + resultTrailingZeroCount, + fastResult); + if (fastResult.fastSignum == 0) { + fastResult.fastScale = 0; + } else { + fastResult.fastScale -= resultTrailingZeroCount; + } + } + + return true; + } + + public static boolean fastMultiply5x5HalfWords( + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + fastMultiply5x5HalfWords( + fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastResult); + } + + /** + * Fast decimal multiplication on two decimals that have been already scaled and whose results + * will fit in 38 digits. + * + * The caller is responsible checking for overflow within the highword and determining + * if scale down appropriate. + * + * @return Returns false if the multiplication resulted in large overflow. Values in result are + * undefined in that case. + */ + public static boolean fastMultiply5x5HalfWords( + long left0, long left1, long left2, + long right0, long right1, long right2, + FastHiveDecimal fastResult) { + + long product; + + final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL; + + final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL; + + // v[0] + product = + halfRight0 * halfLeft0; + final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0]. + product = + halfRight0 + * halfLeft1 + + halfRight1 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[2] + product = + halfRight0 + * halfLeft2 + + halfRight1 + * halfLeft1 + + halfRight2 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[3] + product = + halfRight0 + * halfLeft3 + + halfRight1 + * halfLeft2 + + halfRight2 + * halfLeft1 + + halfRight3 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[4] + product = + halfRight0 + * halfLeft4 + + halfRight1 + * halfLeft3 + + halfRight2 + * halfLeft2 + + halfRight3 + * halfLeft1 + + halfRight4 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + + // v[5] is not calculated since high integer is always 0 for our decimals. + + // These remaining combinations below definitely result in overflow. + if ((halfRight4 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0 || halfLeft1 != 0)) + || (halfRight3 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0)) + || (halfRight2 != 0 && (halfLeft4 != 0 || halfLeft3 != 0)) + || (halfRight1 != 0 && halfLeft4 != 0)) { + return false; + } + + + final long result0 = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0; + final long result1 = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2; + final long result2 = product; + + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastSignum = 0; + } + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + + return true; + } + + public static boolean fastMultiplyFullInternal( + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + long[] result) { + return + fastMultiplyFullInternal( + fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastRight.fast0, fastRight.fast1, fastRight.fast2, + result); + } + + /** + * Fast decimal multiplication on two decimals that have been already scaled and whose results + * will fit in 38 digits. + * + * The caller is responsible checking for overflow within the highword and determining + * if scale down appropriate. + * + * @return Returns false if the multiplication resulted in large overflow. Values in result are + * undefined in that case. + */ + public static boolean fastMultiply5x5HalfWords( + long left0, long left1, long left2, + long right0, long right1, long right2, + long[] result) { + + long product; + + final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL; + + final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL; + + // v[0] + product = + halfRight0 * halfLeft0; + final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0]. + product = + halfRight0 + * halfLeft1 + + halfRight1 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[2] + product = + halfRight0 + * halfLeft2 + + halfRight1 + * halfLeft1 + + halfRight2 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[3] + product = + halfRight0 + * halfLeft3 + + halfRight1 + * halfLeft2 + + halfRight2 + * halfLeft1 + + halfRight3 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[4] + product = + halfRight0 + * halfLeft4 + + halfRight1 + * halfLeft3 + + halfRight2 + * halfLeft2 + + halfRight3 + * halfLeft1 + + halfRight4 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + + // v[5] is not calculated since high integer is always 0 for our decimals. + + // These remaining combinations below definitely result in overflow. + if ((halfRight4 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0 || halfLeft1 != 0)) + || (halfRight3 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0)) + || (halfRight2 != 0 && (halfLeft4 != 0 || halfLeft3 != 0)) + || (halfRight1 != 0 && halfLeft4 != 0)) { + return false; + } + + result[0] = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0; + result[1] = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2; + result[2] = product; + + return true; + } + + /** + * Fast decimal multiplication on two decimals whose results are permitted to go beyond + * 38 digits to the maximum possible 76 digits. The caller is responsible for scaling and + * rounding the results back to 38 or fewer digits. + * + * The caller is responsible for determining the signum. + * + * @param left0 + * @param left1 + * @param left2 + * @param right0 + * @param right1 + * @param right2 + * @param result This full result has 5 longs. + * @return Returns false if the multiplication resulted in an overflow. Values in result are + * undefined in that case. + */ + public static boolean fastMultiplyFullInternal( + long left0, long left1, long left2, + long right0, long right1, long right2, + long[] result) { + assert (result.length == 5); + if (result.length != 5) { + throw new IllegalArgumentException("Expecting result array length = 5"); + } + + long product; + + final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL; + + final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL; + + // v[0] + product = + halfRight0 * halfLeft0; + final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0]. + product = + halfRight0 + * halfLeft1 + + halfRight1 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[2] + product = + halfRight0 + * halfLeft2 + + halfRight1 + * halfLeft1 + + halfRight2 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[3] + product = + halfRight0 + * halfLeft3 + + halfRight1 + * halfLeft2 + + halfRight2 + * halfLeft1 + + halfRight3 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[4] + product = + halfRight0 + * halfLeft4 + + halfRight1 + * halfLeft3 + + halfRight2 + * halfLeft2 + + halfRight3 + * halfLeft1 + + halfRight4 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z4 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[5] -- since integer #5 is always 0, some products here are not included. + product = + halfRight1 + * halfLeft4 + + halfRight2 + * halfLeft3 + + halfRight3 + * halfLeft2 + + halfRight4 + * halfLeft1 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z5 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[6] -- since integer #5 is always 0, some products here are not included. + product = + halfRight2 + * halfLeft4 + + halfRight3 + * halfLeft3 + + halfRight4 + * halfLeft2 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z6 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[7] -- since integer #5 is always 0, some products here are not included. + product = + halfRight3 + * halfLeft4 + + halfRight4 + * halfLeft3 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z7 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[8] -- since integer #5 is always 0, some products here are not included. + product = + halfRight4 + * halfLeft4 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z8 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[9] -- since integer #5 is always 0, some products here are not included. + product = + (product / MULTIPLER_INTWORD_DECIMAL); + if (product > FULL_MAX_HIGHWORD_DECIMAL) { + return false; + } + + result[0] = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0; + result[1] = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2; + result[2] = (long) z5 * MULTIPLER_INTWORD_DECIMAL + (long) z4; + result[3] = (long) z7 * MULTIPLER_INTWORD_DECIMAL + (long) z6; + result[4] = product * MULTIPLER_INTWORD_DECIMAL + (long) z8; + + return true; + } + + /** + * Fast decimal multiplication on two decimals whose results are permitted to go beyond + * 38 digits to the maximum possible 76 digits. The caller is responsible for scaling and + * rounding the results back to 38 or fewer digits. + * + * The caller is responsible for determining the signum. + * + * @param result This full result has 5 longs. + * @return Returns false if the multiplication resulted in an overflow. Values in result are + * undefined in that case. + */ + public static boolean fastMultiply5x6HalfWords( + long left0, long left1, long left2, + long right0, long right1, long right2, + long[] result) { + + if (result.length != 6) { + throw new RuntimeException("Expecting result array length = 6"); + } + + long product; + + final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight5 = right2 / MULTIPLER_INTWORD_DECIMAL; + + final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL; + + // v[0] + product = + halfRight0 * halfLeft0; + final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0]. + product = + halfRight0 + * halfLeft1 + + halfRight1 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[2] + product = + halfRight0 + * halfLeft2 + + halfRight1 + * halfLeft1 + + halfRight2 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[3] + product = + halfRight0 + * halfLeft3 + + halfRight1 + * halfLeft2 + + halfRight2 + * halfLeft1 + + halfRight3 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[4] + product = + halfRight0 + * halfLeft4 + + halfRight1 + * halfLeft3 + + halfRight2 + * halfLeft2 + + halfRight3 + * halfLeft1 + + halfRight4 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z4 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[5] -- since left integer #5 is always 0, some products here are not included. + product = + halfRight1 + * halfLeft4 + + halfRight2 + * halfLeft3 + + halfRight3 + * halfLeft2 + + halfRight4 + * halfLeft1 + + halfRight5 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z5 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[6] -- since left integer #5 is always 0, some products here are not included. + product = + halfRight2 + * halfLeft4 + + halfRight3 + * halfLeft3 + + halfRight4 + * halfLeft2 + + halfRight5 + * halfLeft1 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z6 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[7] -- since left integer #5 is always 0, some products here are not included. + product = + halfRight3 + * halfLeft4 + + halfRight4 + * halfLeft3 + + halfRight5 + * halfLeft2 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z7 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[8] -- since left integer #5 is always 0, some products here are not included. + product = + halfRight4 + * halfLeft4 + + halfRight5 + * halfLeft3 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z8 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[9] -- since left integer #5 is always 0, some products here are not included. + product = + halfRight5 + * halfLeft4 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z9 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[10] -- since left integer #5 is always 0, some products here are not included. + product = + + (product / MULTIPLER_INTWORD_DECIMAL); + if (product > MULTIPLER_INTWORD_DECIMAL) { + return false; + } + + result[0] = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0; + result[1] = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2; + result[2] = (long) z5 * MULTIPLER_INTWORD_DECIMAL + (long) z4; + result[3] = (long) z7 * MULTIPLER_INTWORD_DECIMAL + (long) z6; + result[4] = (long) z9 * MULTIPLER_INTWORD_DECIMAL + (long) z8; + result[5] = product; + + return true; + } + + public static boolean fastMultiply( + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return fastMultiply( + fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastLeft.fastIntegerDigitCount, fastLeft.fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + public static boolean fastMultiply( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + if (leftSignum == 0 || rightSignum == 0) { + fastResult.fastReset(); + return true; + } + + return doMultiply( + leftSignum, leftFast0, leftFast1, leftFast2, + leftIntegerDigitCount, leftScale, + rightSignum, rightFast0, rightFast1, rightFast2, + rightIntegerDigitCount, rightScale, + fastResult); + } + + //************************************************************************************************ + // Decimal Division / Remainder. + + /** + * EXPERMIMENTAL: Division when divisor fits in a single decimal longword. + * + * @return remainderSubexpr2 + */ + private static long doSingleWordQuotient( + long leftFast0, long leftFast1, long leftFast2, + long rightFast0, + FastHiveDecimal fastResult) { + + long quotient2; + long quotient1; + long quotient0; + + long remainderSubexpr2; + + if (leftFast2 == 0 && leftFast1 == 0) { + quotient2 = 0; + quotient1 = 0; + quotient0 = + leftFast0 / rightFast0; + final long k0 = + leftFast0 - quotient0 * rightFast0; + remainderSubexpr2 = + k0 * MULTIPLER_LONGWORD_DECIMAL; + } else if (leftFast2 == 0) { + // leftFast1 != 0. + quotient2 = 0; + quotient1 = + leftFast1 / rightFast0; + final long k1 = + leftFast1 - quotient1 * rightFast0; + final long quotientSubexpr0 = + k1 * MULTIPLER_LONGWORD_DECIMAL + + leftFast0; + quotient0 = + quotientSubexpr0 / rightFast0; + final long k0 = + quotientSubexpr0 - quotient0 * rightFast0; + remainderSubexpr2 = + k0 * MULTIPLER_LONGWORD_DECIMAL; + } else if (leftFast1 == 0){ + // leftFast2 != 0 && leftFast1 == 0. + quotient2 = + leftFast2 / rightFast0; + quotient1 = 0; + quotient0 = + leftFast0 / rightFast0; + final long k0 = + leftFast0 - quotient0 * rightFast0; + remainderSubexpr2 = + k0 * MULTIPLER_LONGWORD_DECIMAL; + } else { + quotient2 = + leftFast2 / rightFast0; + final long k2 = + leftFast2 - quotient2 * rightFast0; + final long quotientSubexpr1 = + k2 * MULTIPLER_LONGWORD_DECIMAL + + leftFast1; + quotient1 = + quotientSubexpr1 / rightFast0; + final long k1 = + quotientSubexpr1 - quotient1 * rightFast0; + final long quotientSubexpr0 = + k1 * MULTIPLER_LONGWORD_DECIMAL; + quotient0 = + quotientSubexpr0 / rightFast0; + final long k0 = + quotientSubexpr0 - quotient0 * rightFast0; + remainderSubexpr2 = + k0 * MULTIPLER_LONGWORD_DECIMAL; + } + + fastResult.fast0 = quotient0; + fastResult.fast1 = quotient1; + fastResult.fast2 = quotient2; + + return remainderSubexpr2; + } + + private static int doSingleWordRemainder( + long leftFast0, long leftFast1, long leftFast2, + long rightFast0, + long remainderSubexpr2, + FastHiveDecimal fastResult) { + + int remainderDigitCount; + + long remainder2; + long remainder1; + long remainder0; + + if (remainderSubexpr2 == 0) { + remainder2 = 0; + remainder1 = 0; + remainder0 = 0; + remainderDigitCount = 0; + } else { + remainder2 = + remainderSubexpr2 / rightFast0; + final long k2 = + remainderSubexpr2 - remainder2 * rightFast0; + if (k2 == 0) { + remainder1 = 0; + remainder0 = 0; + remainderDigitCount = + LONGWORD_DECIMAL_DIGITS - fastLongWordTrailingZeroCount(remainder2); + } else { + final long remainderSubexpr1 = + k2 * MULTIPLER_LONGWORD_DECIMAL; + long remainderSubexpr0; + remainder1 = + remainderSubexpr1 / rightFast0; + final long k1 = + remainderSubexpr1 - remainder1 * rightFast0; + if (k1 == 0) { + remainder0 = 0; + remainderDigitCount = + LONGWORD_DECIMAL_DIGITS + + LONGWORD_DECIMAL_DIGITS - fastLongWordTrailingZeroCount(remainder1); + } else { + remainderSubexpr0 = + k2 * MULTIPLER_LONGWORD_DECIMAL; + + remainder0 = + remainderSubexpr0 / rightFast0; + remainderDigitCount = + TWO_X_LONGWORD_DECIMAL_DIGITS + + LONGWORD_DECIMAL_DIGITS - fastLongWordTrailingZeroCount(remainder0); + } + } + } + + fastResult.fast0 = remainder0; + fastResult.fast1 = remainder1; + fastResult.fast2 = remainder2; + + return remainderDigitCount; + } + + // EXPERIMENT + private static boolean fastSingleWordDivision( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, int leftScale, + int rightSignum, long rightFast0, int rightScale, + FastHiveDecimal fastResult) { + + long remainderSubexpr2 = + doSingleWordQuotient( + leftFast0, leftFast1, leftFast2, + rightFast0, + fastResult); + + long quotient0 = fastResult.fast0; + long quotient1 = fastResult.fast1; + long quotient2 = fastResult.fast2; + + int quotientDigitCount; + if (quotient2 != 0) { + quotientDigitCount = fastLongWordPrecision(quotient2); + } else if (quotient1 != 0) { + quotientDigitCount = fastLongWordPrecision(quotient1); + } else { + quotientDigitCount = fastLongWordPrecision(quotient0); + } + + int remainderDigitCount = + doSingleWordRemainder( + leftFast0, leftFast1, leftFast2, + rightFast0, + remainderSubexpr2, + fastResult); + + long remainder0 = fastResult.fast0; + long remainder1 = fastResult.fast1; + long remainder2 = fastResult.fast2; + + fastResult.fast0 = quotient0; + fastResult.fast1 = quotient1; + fastResult.fast2 = quotient2; + + final int quotientScale = leftScale + rightScale; + + if (remainderDigitCount == 0) { + fastResult.fastScale = quotientScale; + } else { + int resultScale = quotientScale + remainderDigitCount; + + int adjustedQuotientDigitCount; + if (quotientScale > 0) { + adjustedQuotientDigitCount = Math.max(0, quotientDigitCount - quotientScale); + } else { + adjustedQuotientDigitCount = quotientDigitCount; + } + final int maxScale = HiveDecimal.MAX_SCALE - adjustedQuotientDigitCount; + + int scale = Math.min(resultScale, maxScale); + + int remainderScale; + remainderScale = Math.min(remainderDigitCount, maxScale - quotientScale); + if (remainderScale > 0) { + if (quotientDigitCount > 0) { + // Make room for remainder. + fastScaleUp( + fastResult, + remainderScale, + fastResult); + } + // Copy in remainder digits... which start at the top of remainder2. + if (remainderScale < LONGWORD_DECIMAL_DIGITS) { + final long remainderDivisor2 = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - remainderScale]; + fastResult.fast0 += (remainder2 / remainderDivisor2); + } else if (remainderScale == LONGWORD_DECIMAL_DIGITS) { + fastResult.fast0 = remainder2; + } else if (remainderScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + final long remainderDivisor2 = powerOfTenTable[remainderScale - LONGWORD_DECIMAL_DIGITS]; + fastResult.fast1 += (remainder2 / remainderDivisor2); + fastResult.fast0 = remainder1; + } else if (remainderScale == TWO_X_LONGWORD_DECIMAL_DIGITS) { + fastResult.fast1 = remainder2; + fastResult.fast0 = remainder1; + } + } + + // UNDONE: Method is still under development. + fastResult.fastScale = scale; + + // UNDONE: Trim trailing zeroes... + } + + return true; + } + + public static boolean fastDivide( + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return fastDivide( + fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastLeft.fastIntegerDigitCount, fastLeft.fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + public static boolean fastDivide( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + // Arithmetic operations reset the results. + fastResult.fastReset(); + + if (rightSignum == 0) { + // Division by 0. + return false; + } + if (leftSignum == 0) { + // Zero result. + return true; + } + + /* + if (rightFast1 == 0 && rightFast2 == 0) { + return fastSingleWordDivision( + leftSignum, leftFast0, leftFast1, leftFast2, leftScale, + rightSignum, rightFast0, rightScale, + fastResult); + } + */ + + BigDecimal denominator = + fastBigDecimalValue( + leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale); + BigDecimal divisor = + fastBigDecimalValue( + rightSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale); + BigDecimal quotient = + denominator.divide(divisor, HiveDecimal.MAX_SCALE, BigDecimal.ROUND_HALF_UP); + + if (!fastSetFromBigDecimal( + quotient, + true, + fastResult)) { + return false; + } + + return true; + } + + public static boolean fastRemainder( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + // Arithmetic operations reset the results. + fastResult.fastReset(); + + if (rightSignum == 0) { + // Division by 0. + return false; + } + if (leftSignum == 0) { + // Zero result. + return true; + } + + BigDecimal denominator = + fastBigDecimalValue( + leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale); + BigDecimal divisor = + fastBigDecimalValue( + rightSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale); + BigDecimal remainder = + denominator.remainder(divisor); + fastResult.fastReset(); + if (!fastSetFromBigDecimal( + remainder, + true, + fastResult)) { + return false; + } + + return true; + } + + public static boolean fastPow( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int exponent, + FastHiveDecimal fastResult) { + + // Arithmetic operations (re)set the results. + fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + + if (exponent < 0) { + // UNDONE: Currently, negative exponent is not supported. + return false; + } + + for (int e = 1; e < exponent; e++) { + if (!doMultiply( + fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale, + fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale, + fastResult)) { + return false; + } + } + return true; + } + + //************************************************************************************************ + // Decimal String Formatting. + + public static String fastToFormatString( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int formatScale) { + byte[] scratchBuffer = new byte[FAST_SCRATCH_BUFFER_LEN_TO_BYTES]; + final int index = + doFastToFormatBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + formatScale, + scratchBuffer); + return + new String(scratchBuffer, index, FAST_SCRATCH_BUFFER_LEN_TO_BYTES - index); + } + + //************************************************************************************************ + // Decimal String Formatting. + + public static String fastToFormatString( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int formatScale, + byte[] scratchBuffer) { + final int index = + doFastToBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, formatScale, + scratchBuffer); + return new String(scratchBuffer, index, scratchBuffer.length - index); + } + + public static int fastToFormatBytes( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int formatScale, + byte[] scratchBuffer) { + return + doFastToFormatBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + formatScale, + scratchBuffer); + } + + public static int doFastToFormatBytes( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int formatScale, + byte[] scratchBuffer) { + + // NOTE: OldHiveDecimal.toFormatString returns decimal strings with more than > 38 digits! + + if (formatScale >= fastScale) { + return + doFastToBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, formatScale, + scratchBuffer); + } else { + FastHiveDecimal fastTemp = new FastHiveDecimal(); + if (!fastRound( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + formatScale, BigDecimal.ROUND_HALF_UP, + fastTemp)) { + return 0; + } + return + doFastToBytes( + fastTemp.fastSignum, fastTemp.fast0, fastTemp.fast1, fastTemp.fast2, + fastTemp.fastIntegerDigitCount, fastTemp.fastScale, formatScale, + scratchBuffer); + } + } + + public static String fastToString( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale) { + return doFastToString( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, fastTrailingZeroesScale); + } + + public static String fastToString( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale, + byte[] scratchBuffer) { + return doFastToString( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, fastTrailingZeroesScale, + scratchBuffer); + } + + public static String fastToDigitsOnlyString( + long fast0, long fast1, long fast2, + int fastIntegerDigitCount) { + byte[] scratchBuffer = new byte[FAST_SCRATCH_BUFFER_LEN_TO_BYTES]; + final int index = + doFastToDigitsOnlyBytes( + fast0, fast1, fast2, + fastIntegerDigitCount, + scratchBuffer); + return + new String(scratchBuffer, index, FAST_SCRATCH_BUFFER_LEN_TO_BYTES - index); + } + + public static int fastToBytes( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale, + byte[] scratchBuffer) { + return doFastToBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, fastTrailingZeroesScale, + scratchBuffer); + } + + private static String doFastToString( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale) { + byte[] scratchBuffer = new byte[FAST_SCRATCH_BUFFER_LEN_TO_BYTES]; + final int index = + doFastToBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, fastTrailingZeroesScale, + scratchBuffer); + return + new String( + scratchBuffer, index, FAST_SCRATCH_BUFFER_LEN_TO_BYTES - index); + } + + private static String doFastToString( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale, + byte[] scratchBuffer) { + final int index = + doFastToBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, fastTrailingZeroesScale, + scratchBuffer); + return new String(scratchBuffer, index, scratchBuffer.length - index); + } + + private static int doFastToBytes( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale, + byte[] scratchBuffer) { + + int index = scratchBuffer.length - 1; + + int trailingZeroCount = + (fastTrailingZeroesScale != -1 ? fastTrailingZeroesScale - fastScale : 0); + // Virtual trailing zeroes. + if (trailingZeroCount > 0) { + for (int i = 0; i < trailingZeroCount; i++) { + scratchBuffer[index--] = BYTE_DIGIT_ZERO; + } + } + + // Scale fractional digits, dot, integer digits. + + final int scale = fastScale; + + final boolean isZeroFast1AndFast2 = (fast1 == 0 && fast2 == 0); + final boolean isZeroFast2 = (fast2 == 0); + + int lowerLongwordScale = 0; + int middleLongwordScale = 0; + int highLongwordScale = 0; + long longWord = fast0; + if (scale > 0) { + + // Fraction digits from lower longword. + + lowerLongwordScale = Math.min(scale, LONGWORD_DECIMAL_DIGITS); + + for (int i = 0; i < lowerLongwordScale; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + longWord /= 10; + } + if (lowerLongwordScale == LONGWORD_DECIMAL_DIGITS) { + longWord = fast1; + } + + if (scale > LONGWORD_DECIMAL_DIGITS) { + + // Fraction digits continue into middle longword. + + middleLongwordScale = Math.min(scale - LONGWORD_DECIMAL_DIGITS, LONGWORD_DECIMAL_DIGITS); + for (int i = 0; i < middleLongwordScale; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + longWord /= 10; + } + if (middleLongwordScale == LONGWORD_DECIMAL_DIGITS) { + longWord = fast2; + } + + if (scale > TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Fraction digit continue into highest longword. + + highLongwordScale = scale - TWO_X_LONGWORD_DECIMAL_DIGITS; + for (int i = 0; i < highLongwordScale; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + longWord /= 10; + } + } + } + scratchBuffer[index--] = BYTE_DOT; + } else if (trailingZeroCount > 0) { + scratchBuffer[index--] = BYTE_DOT; + } + + // Integer digits; stop on zeroes above. + + boolean atLeastOneIntegerDigit = false; + if (scale <= LONGWORD_DECIMAL_DIGITS) { + + // Handle remaining lower long word digits as integer digits. + + final int remainingLowerLongwordDigits = LONGWORD_DECIMAL_DIGITS - lowerLongwordScale; + for (int i = 0; i < remainingLowerLongwordDigits; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + atLeastOneIntegerDigit = true; + longWord /= 10; + if (longWord == 0 && isZeroFast1AndFast2) { + // Suppress leading zeroes. + break; + } + } + if (isZeroFast1AndFast2) { + if (!atLeastOneIntegerDigit) { + scratchBuffer[index--] = BYTE_DIGIT_ZERO; + } + if (fastSignum == -1) { + scratchBuffer[index--] = BYTE_MINUS; + } + return index + 1; + } + longWord = fast1; + } + + if (scale <= TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Handle remaining middle long word digits. + + final int remainingMiddleLongwordDigits = LONGWORD_DECIMAL_DIGITS - middleLongwordScale; + + for (int i = 0; i < remainingMiddleLongwordDigits; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + atLeastOneIntegerDigit = true; + longWord /= 10; + if (longWord == 0 && isZeroFast2) { + // Suppress leading zeroes. + break; + } + } + if (isZeroFast2) { + if (!atLeastOneIntegerDigit) { + scratchBuffer[index--] = BYTE_DIGIT_ZERO; + } + if (fastSignum == -1) { + scratchBuffer[index--] = BYTE_MINUS; + } + return index + 1; + } + longWord = fast2; + } + + final int remainingHighwordDigits = HIGHWORD_DECIMAL_DIGITS - highLongwordScale; + + for (int i = 0; i < remainingHighwordDigits; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + atLeastOneIntegerDigit = true; + longWord /= 10; + if (longWord == 0) { + // Suppress leading zeroes. + break; + } + } + if (!atLeastOneIntegerDigit) { + scratchBuffer[index--] = BYTE_DIGIT_ZERO; + } + if (fastSignum == -1) { + scratchBuffer[index--] = BYTE_MINUS; + } + return index + 1; + } + + public static int fastToDigitsOnlyBytes( + long fast0, long fast1, long fast2, + int fastIntegerDigitCount, + byte[] scratchBuffer) { + return doFastToDigitsOnlyBytes( + fast0, fast1, fast2, + fastIntegerDigitCount, + scratchBuffer); + } + + private static int doFastToDigitsOnlyBytes( + long fast0, long fast1, long fast2, + int fastIntegerDigitCount, + byte[] scratchBuffer) { + + int index = scratchBuffer.length - 1; + + // Just digits. + + final boolean isZeroFast1AndFast2 = (fast1 == 0 && fast2 == 0); + final boolean isZeroFast2 = (fast2 == 0); + + boolean atLeastOneIntegerDigit = false; + long longWord = fast0; + for (int i = 0; i < LONGWORD_DECIMAL_DIGITS; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + atLeastOneIntegerDigit = true; + longWord /= 10; + if (longWord == 0 && isZeroFast1AndFast2) { + // Suppress leading zeroes. + break; + } + } + if (isZeroFast1AndFast2) { + if (!atLeastOneIntegerDigit) { + scratchBuffer[index--] = BYTE_DIGIT_ZERO; + } + return index + 1; + } + + longWord = fast1; + + for (int i = 0; i < LONGWORD_DECIMAL_DIGITS; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + atLeastOneIntegerDigit = true; + longWord /= 10; + if (longWord == 0 && isZeroFast2) { + // Suppress leading zeroes. + break; + } + } + if (isZeroFast2) { + if (!atLeastOneIntegerDigit) { + scratchBuffer[index--] = BYTE_DIGIT_ZERO; + } + return index + 1; + } + + longWord = fast2; + + for (int i = 0; i < HIGHWORD_DECIMAL_DIGITS; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + atLeastOneIntegerDigit = true; + longWord /= 10; + if (longWord == 0) { + // Suppress leading zeroes. + break; + } + } + if (!atLeastOneIntegerDigit) { + scratchBuffer[index--] = BYTE_DIGIT_ZERO; + } + return index + 1; + } + + //************************************************************************************************ + // Decimal Validation. + + public static boolean fastIsValid(FastHiveDecimal fastDec) { + return fastIsValid( + fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2, + fastDec.fastIntegerDigitCount, fastDec.fastScale); + } + + public static boolean fastIsValid( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + boolean isValid; + if (fastSignum == 0) { + isValid = (fast0 == 0 && fast1 == 0 && fast2 == 0 && fastIntegerDigitCount == 0 && fastScale == 0); + if (!isValid) { + System.out.println("FAST_IS_VALID signum 0 but other fields not"); + } + } else { + isValid = ( + (fast0 >= 0 && fast0 <= MAX_LONGWORD_DECIMAL) && + (fast1 >= 0 && fast1 <= MAX_LONGWORD_DECIMAL) && + (fast2 >= 0 && fast2 <= MAX_HIGHWORD_DECIMAL)); + if (!isValid) { + System.out.println("FAST_IS_VALID fast0 .. fast2 out of range"); + } else { + if (fastScale < 0 || fastScale > HiveDecimal.MAX_SCALE) { + System.out.println("FAST_IS_VALID fastScale " + fastScale + " out of range"); + isValid = false; + } else if (fastIntegerDigitCount < 0 || fastIntegerDigitCount > HiveDecimal.MAX_PRECISION) { + System.out.println("FAST_IS_VALID fastIntegerDigitCount " + fastIntegerDigitCount + " out of range"); + isValid = false; + } else if (fastIntegerDigitCount + fastScale > HiveDecimal.MAX_PRECISION) { + System.out.println("FAST_IS_VALID exceeds max precision: fastIntegerDigitCount " + fastIntegerDigitCount + " and fastScale " + fastScale); + isValid = false; + } else { + // Verify integerDigitCount given fastScale. + final int rawPrecision = fastRawPrecision(fastSignum, fast0, fast1, fast2); + if (fastIntegerDigitCount > 0) { + if (rawPrecision != fastIntegerDigitCount + fastScale) { + System.out.println("FAST_IS_VALID integer case: rawPrecision " + rawPrecision + + " fastIntegerDigitCount " + fastIntegerDigitCount + + " fastScale " + fastScale); + isValid = false; + } + } else { + if (rawPrecision > fastScale) { + System.out.println("FAST_IS_VALID fraction only case: rawPrecision " + rawPrecision + + " fastIntegerDigitCount " + fastIntegerDigitCount + + " fastScale " + fastScale); + isValid = false; + } + } + if (isValid) { + final int trailingZeroCount = + fastTrailingDecimalZeroCount( + fast0, fast1, fast2, + fastIntegerDigitCount, fastScale); + if (trailingZeroCount != 0) { + System.out.println("FAST_IS_VALID exceeds max precision: trailingZeroCount != 0"); + isValid = false; + } + } + } + } + } + + if (!isValid) { + System.out.println("FAST_IS_VALID fast0 " + fast0); + System.out.println("FAST_IS_VALID fast1 " + fast1); + System.out.println("FAST_IS_VALID fast2 " + fast2); + System.out.println("FAST_IS_VALID fastIntegerDigitCount " + fastIntegerDigitCount); + System.out.println("FAST_IS_VALID fastScale " + fastScale); + } + return isValid; + } + + public static void fastRaiseInvalidException( + FastHiveDecimal fastResult) { + throw new RuntimeException( + "Invalid fast decimal " + + " fastSignum " + fastResult.fastSignum + " fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2 + + " fastIntegerDigitCount " + fastResult.fastIntegerDigitCount + " fastScale " + fastResult.fastScale + + " stack trace: " + getStackTraceAsSingleLine(Thread.currentThread().getStackTrace())); + } + + public static void fastRaiseInvalidException( + FastHiveDecimal fastResult, + String parameters) { + throw new RuntimeException( + "Parameters: " + parameters + " --> " + + "Invalid fast decimal " + + " fastSignum " + fastResult.fastSignum + " fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2 + + " fastIntegerDigitCount " + fastResult.fastIntegerDigitCount + " fastScale " + fastResult.fastScale + + " stack trace: " + getStackTraceAsSingleLine(Thread.currentThread().getStackTrace())); + } + + //************************************************************************************************ + // Decimal Debugging. + + static final int STACK_LENGTH_LIMIT = 20; + public static String getStackTraceAsSingleLine(StackTraceElement[] stackTrace) { + StringBuilder sb = new StringBuilder(); + sb.append("Stack trace: "); + int length = stackTrace.length; + boolean isTruncated = false; + if (length > STACK_LENGTH_LIMIT) { + length = STACK_LENGTH_LIMIT; + isTruncated = true; + } + for (int i = 0; i < length; i++) { + if (i > 0) { + sb.append(", "); + } + sb.append(stackTrace[i]); + } + if (isTruncated) { + sb.append(", ..."); + } + + return sb.toString(); + } + + public static String displayBytes(byte[] bytes, int start, int length) { + StringBuilder sb = new StringBuilder(); + for (int i = start; i < start + length; i++) { + sb.append(String.format("\\%03d", (int) (bytes[i] & 0xff))); + } + return sb.toString(); + } +} \ No newline at end of file diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java new file mode 100644 index 0000000000000000000000000000000000000000..53684e7ab1b8d473ac7618ab2c830c226175a18c --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import org.apache.commons.lang.StringUtils; + +public abstract class HiveBaseChar { + protected String value; + + protected HiveBaseChar() { + } + + /** + * Sets the string value to a new value, obeying the max length defined for this object. + * @param val new value + */ + public void setValue(String val, int maxLength) { + value = HiveBaseChar.enforceMaxLength(val, maxLength); + } + + public void setValue(HiveBaseChar val, int maxLength) { + setValue(val.value, maxLength); + } + + public static String enforceMaxLength(String val, int maxLength) { + if (val == null) { + return null; + } + String value = val; + + if (maxLength > 0) { + int valLength = val.codePointCount(0, val.length()); + if (valLength > maxLength) { + // Truncate the excess chars to fit the character length. + // Also make sure we take supplementary chars into account. + value = val.substring(0, val.offsetByCodePoints(0, maxLength)); + } + } + return value; + } + + public static String getPaddedValue(String val, int maxLength) { + if (val == null) { + return null; + } + if (maxLength < 0) { + return val; + } + + int valLength = val.codePointCount(0, val.length()); + if (valLength > maxLength) { + return enforceMaxLength(val, maxLength); + } + + if (maxLength > valLength) { + // Make sure we pad the right amount of spaces; valLength is in terms of code points, + // while StringUtils.rpad() is based on the number of java chars. + int padLength = val.length() + (maxLength - valLength); + val = StringUtils.rightPad(val, padLength); + } + return val; + } + + public String getValue() { + return value; + } + + public int getCharacterLength() { + return value.codePointCount(0, value.length()); + } + + @Override + public int hashCode() { + return getValue().hashCode(); + } + + @Override + public String toString() { + return getValue(); + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveChar.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveChar.java new file mode 100644 index 0000000000000000000000000000000000000000..66aa524bb2c3b0835cdb8af3001ec6d96b5ae67c --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveChar.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import org.apache.commons.lang.StringUtils; + +/** + * HiveChar. + * String values will be padded to full char length. + * Character legnth, comparison, hashCode should ignore trailing spaces. + */ +public class HiveChar extends HiveBaseChar + implements Comparable { + + public static final int MAX_CHAR_LENGTH = 255; + + public HiveChar() { + } + + public HiveChar(String val, int len) { + setValue(val, len); + } + + public HiveChar(HiveChar hc, int len) { + setValue(hc.value, len); + } + + /** + * Set char value, padding or truncating the value to the size of len parameter. + */ + public void setValue(String val, int len) { + super.setValue(HiveBaseChar.getPaddedValue(val, len), -1); + } + + public void setValue(String val) { + setValue(val, -1); + } + + public String getStrippedValue() { + return StringUtils.stripEnd(value, " "); + } + + public String getPaddedValue() { + return value; + } + + public int getCharacterLength() { + String strippedValue = getStrippedValue(); + return strippedValue.codePointCount(0, strippedValue.length()); + } + + public String toString() { + return getPaddedValue(); + } + + public int compareTo(HiveChar rhs) { + if (rhs == this) { + return 0; + } + return this.getStrippedValue().compareTo(rhs.getStrippedValue()); + } + + public boolean equals(Object rhs) { + if (rhs == this) { + return true; + } + if (rhs == null || rhs.getClass() != getClass()) { + return false; + } + return this.getStrippedValue().equals(((HiveChar) rhs).getStrippedValue()); + } + + public int hashCode() { + return getStrippedValue().hashCode(); + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java new file mode 100644 index 0000000000000000000000000000000000000000..690e95762444a17fa601e03f68302fe8903c62a3 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java @@ -0,0 +1,1506 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import java.util.Arrays; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.BigInteger; + +/** + * HiveDecimal is a decimal data type with a maximum precision and scale. + *

+ * It is the Hive DECIMAL data type. + *

+ * The scale is the number of fractional decimal digits. The digits after the dot. It is limited + * to 38 (MAX_SCALE). + *

+ * The precision is the integer (or whole-number) decimal digits plus fractional decimal digits. + * It is limited to a total of 38 digits (MAX_PRECISION). + *

+ * Hive syntax for declaring DECIMAL has 3 forms: + *

+ * {@code + * DECIMAL // Use the default precision/scale.} + *

+ * {@code + * DECIMAL(precision) // Use the default scale.} + *

+ * {@code + * DECIMAL(precision, scale)} + * } + *

+ * The declared scale must be <= precision. + *

+ * Use DECIMAL instead of DOUBLE when exact numeric accuracy is required. Not all decimal numbers + * (radix 10) are exactly representable in the binary (radix 2 based) floating point type DOUBLE and + * cause accuracy anomalies (i.e. wrong results). See the Internet for more details. + *

+ * HiveDecimal is implemented as a classic Java immutable object. All operations on HiveDecimal + * that produce a different value will create a new HiveDecimal object. + *

+ * Decimals are physically stored without any extra leading or trailing zeroes. The scale of + * a decimal is the number of non-trailing zero fractional digits. + *

+ * Math operations on decimals typically cause the scale to change as a result of the math and + * from trailing fractional digit elimination. + *

+ * Typically, Hive, when it wants to make sure a result decimal fits in the column decimal's + * precision/scale it calls enforcePrecisionScale. That method will scale down or trim off + * result fractional digits if necessary with rounding when the column has a smaller scale. + * And, it will also indicate overflow when the decimal has exceeded the column's maximum precision. + *

+ * NOTE: When Hive gets ready to serialize a decimal into text or binary, it usually sometimes + * wants trailing fractional zeroes. See the special notes for toFormatString and + * bigIntegerBytesScaled for details. + *

+ * ------------------------------------- Version 2 ------------------------------------------------ + *

+ * This is the 2nd major version of HiveDecimal called V2. The previous version has been + * renamed to HiveDecimalV1 and is kept as a test and behavior reference. + *

+ * For good performance we do not represent the decimal using a BigDecimal object like the previous + * version V1 did. Using Java objects to represent our decimal incurs too high a penalty + * for memory allocations and general logic. + *

+ * The original V1 public methods and fields are annotated with @HiveDecimalVersionV1; new public + * methods and fields are annotated with @HiveDecimalVersionV2. + * + */ +public final class HiveDecimal extends FastHiveDecimal implements Comparable { + + /* + * IMPLEMENTATION NOTE: + * We implement HiveDecimal with the mutable FastHiveDecimal class. That class uses + * protected on all its methods so they will not be visible in the HiveDecimal class. + * + * So even if one casts to FastHiveDecimal, you shouldn't be able to violate the immutability + * of a HiveDecimal class. + */ + + //TODO : We should ideally get the values of these constants from serdeConstants for consistency + @HiveDecimalVersionV1 + public static final int MAX_PRECISION = 38; + @HiveDecimalVersionV1 + public static final int MAX_SCALE = 38; + + /** + * Default precision/scale when user doesn't specify in the column metadata, such as + * decimal and decimal(8). + */ + @HiveDecimalVersionV1 + public static final int USER_DEFAULT_PRECISION = 10; + @HiveDecimalVersionV1 + public static final int USER_DEFAULT_SCALE = 0; + + /** + * Default precision/scale when system is not able to determine them, such as in case + * of a non-generic udf. + */ + @HiveDecimalVersionV1 + public static final int SYSTEM_DEFAULT_PRECISION = 38; + @HiveDecimalVersionV1 + public static final int SYSTEM_DEFAULT_SCALE = 18; + + /** + * Common values. + */ + @HiveDecimalVersionV1 + public static final HiveDecimal ZERO = HiveDecimal.create(0); + @HiveDecimalVersionV1 + public static final HiveDecimal ONE = HiveDecimal.create(1); + + /** + * ROUND_FLOOR: + *

+ * Round towards negative infinity. + *

+ * The Hive function is FLOOR. + *

+ * Positive numbers: The round fraction is thrown away. + *

+ * (Example here rounds at scale 0) + * Value FLOOR + * 0.3 0 + * 2 2 + * 2.1 2 + *

+ * Negative numbers: If there is a round fraction, throw it away and subtract 1. + *

+ * (Example here rounds at scale 0) + * Value FLOOR + * -0.3 -1 + * -2 -2 + * -2.1 -3 + */ + @HiveDecimalVersionV1 + public static final int ROUND_FLOOR = BigDecimal.ROUND_FLOOR; + + /** + * ROUND_CEILING: + *

+ * Round towards positive infinity. + *

+ * The Hive function is CEILING. + *

+ * Positive numbers: If there is a round fraction, throw it away and add 1 + *

+ * (Example here rounds at scale 0) + * Value CEILING + * 0.3 1 + * 2 2 + * 2.1 3 + *

+ * Negative numbers: The round fraction is thrown away. + *

+ * (Example here rounds at scale 0) + * Value CEILING + * -0.3 0 + * -2 -2 + * -2.1 -2 + */ + @HiveDecimalVersionV1 + public static final int ROUND_CEILING = BigDecimal.ROUND_CEILING; + + /** + * ROUND_HALF_UP: + *

+ * Round towards "nearest neighbor" unless both neighbors are equidistant then round up. + *

+ * The Hive function is ROUND. + *

+ * For result, throw away round fraction. If the round fraction is >= 0.5, then add 1 when + * positive and subtract 1 when negative. So, the sign is irrelevant. + *

+ * (Example here rounds at scale 0) + * Value ROUND Value ROUND + * 0.3 0 -0.3 0 + * 2 2 -2 -2 + * 2.1 2 -2.1 -2 + * 2.49 2 -2.49 -2 + * 2.5 3 -2.5 -3 + * + */ + @HiveDecimalVersionV1 + public static final int ROUND_HALF_UP = BigDecimal.ROUND_HALF_UP; + + /** + * ROUND_HALF_EVEN: + * Round towards the "nearest neighbor" unless both neighbors are equidistant, then round + * towards the even neighbor. + *

+ * The Hive function is BROUND. + *

+ * Known as Banker’s Rounding. + *

+ * When you add values rounded with ROUND_HALF_UP you have a bias that grows as you add more + * numbers. Banker's Rounding is a way to minimize that bias. It rounds toward the nearest + * even number when the fraction is 0.5 exactly. In table below, notice that 2.5 goes DOWN to + * 2 (even) but 3.5 goes UP to 4 (even), etc. + *

+ * So, the sign is irrelevant. + *

+ * (Example here rounds at scale 0) + * Value BROUND Value BROUND + * 0.49 0 -0.49 0 + * 0.5 0 -0.5 0 + * 0.51 1 -0.51 -1 + * 1.5 2 -1.5 -2 + * 2.5 2 -2.5 -2 + * 2.51 3 -2.51 -3 + * 3.5 4 -3.5 -4 + * 4.5 4 -4.5 -4 + * 4.51 5 -4.51 -5 + * + */ + @HiveDecimalVersionV1 + public static final int ROUND_HALF_EVEN = BigDecimal.ROUND_HALF_EVEN; + + //----------------------------------------------------------------------------------------------- + // Constructors are marked private; use create methods. + //----------------------------------------------------------------------------------------------- + + private HiveDecimal() { + super(); + } + + private HiveDecimal(HiveDecimal dec) { + super(dec); + } + + private HiveDecimal(FastHiveDecimal fastDec) { + super(fastDec); + } + + private HiveDecimal(int fastSignum, FastHiveDecimal fastDec) { + super(fastSignum, fastDec); + } + + private HiveDecimal( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + super(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + //----------------------------------------------------------------------------------------------- + // Create methods. + //----------------------------------------------------------------------------------------------- + + /** + * Create a HiveDecimal from a FastHiveDecimal object. Used by HiveDecimalWritable. + * @param fastDec the value to set + * @return new hive decimal + */ + @HiveDecimalVersionV2 + public static HiveDecimal createFromFast(FastHiveDecimal fastDec) { + return new HiveDecimal(fastDec); + } + + /** + * Create a HiveDecimal from BigDecimal object. + * + * A BigDecimal object has a decimal scale. + * + * We will have overflow if BigDecimal's integer part exceed MAX_PRECISION digits or + * 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1. + * + * When the BigDecimal value's precision exceeds MAX_PRECISION and there are fractional digits + * because of scale > 0, then lower digits are trimmed off with rounding to meet the + * MAX_PRECISION requirement. + * + * Also, BigDecimal supports negative scale -- which means multiplying the value by 10^abs(scale). + * And, BigDecimal allows for a non-zero scale for zero. We normalize that so zero always has + * scale 0. + * + * @param bigDecimal the value to set + * @return The HiveDecimal with the BigDecimal's value adjusted down to a maximum precision. + * Otherwise, null is returned for overflow. + */ + @HiveDecimalVersionV1 + public static HiveDecimal create(BigDecimal bigDecimal) { + return create(bigDecimal, true); + } + + /** + * Same as the above create method, except fractional digit rounding can be turned off. + * @param bigDecimal the value to set + * @param allowRounding True requires all of the bigDecimal value be converted to the decimal + * without loss of precision. + * @return + */ + @HiveDecimalVersionV1 + public static HiveDecimal create(BigDecimal bigDecimal, boolean allowRounding) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBigDecimal( + bigDecimal, allowRounding)) { + return null; + } + return result; + } + + /** + * Creates a HiveDecimal from a BigInteger's value with a scale of 0. + * + * We will have overflow if BigInteger exceed MAX_PRECISION digits or + * 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1. + * + * @param bigInteger the value to set + * @return A HiveDecimal object with the exact BigInteger's value. + * Otherwise, null is returned on overflow. + */ + @HiveDecimalVersionV1 + public static HiveDecimal create(BigInteger bigInteger) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBigInteger( + bigInteger)) { + return null; + } + return result; + } + + /** + * Creates a HiveDecimal from a BigInteger's value with a specified scale. + * + * We will have overflow if BigInteger exceed MAX_PRECISION digits or + * 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1. + * + * The resulting decimal will have fractional digits when the specified scale is greater than 0. + * + * When the BigInteger's value's precision exceeds MAX_PRECISION and there are fractional digits + * because of scale > 0, then lower digits are trimmed off with rounding to meet the + * MAX_PRECISION requirement. + * + * @param bigInteger the value to set + * @param scale the scale to set + * @return A HiveDecimal object with the BigInteger's value adjusted for scale. + * Otherwise, null is returned on overflow. + */ + @HiveDecimalVersionV1 + public static HiveDecimal create(BigInteger bigInteger, int scale) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBigIntegerAndScale( + bigInteger, scale)) { + return null; + } + return result; + } + + /** + * Create a HiveDecimal by parsing a whole string. + * + * We support parsing a decimal with an exponent because the previous version + * (i.e. OldHiveDecimal) uses the BigDecimal parser and was able to. + * + * @param string the string to parse + * @return a new hive decimal + */ + @HiveDecimalVersionV1 + public static HiveDecimal create(String string) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromString( + string, true)) { + return null; + } + return result; + } + + /** + * Same as the method above, except blanks before and after are tolerated. + * @param string the string to parse + * @param trimBlanks True specifies leading and trailing blanks are to be ignored. + * @return a new hive decimal + */ + @HiveDecimalVersionV2 + public static HiveDecimal create(String string, boolean trimBlanks) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromString( + string, trimBlanks)) { + return null; + } + return result; + } + + /** + * Create a HiveDecimal by parsing the characters in a whole byte array. + * + * Same rules as create(String string) above. + * + */ + @HiveDecimalVersionV2 + public static HiveDecimal create(byte[] bytes) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBytes( + bytes, 0, bytes.length, false)) { + return null; + } + return result; + } + + /** + * Same as the method above, except blanks before and after are tolerated. + * + */ + @HiveDecimalVersionV2 + public static HiveDecimal create(byte[] bytes, boolean trimBlanks) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBytes( + bytes, 0, bytes.length, trimBlanks)) { + return null; + } + return result; + } + + /** + * This method takes in digits only UTF-8 characters, a sign flag, and a scale and returns + * a decimal. + */ + @HiveDecimalVersionV2 + public static HiveDecimal create(boolean isNegative, byte[] bytes, int scale) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromDigitsOnlyBytesAndScale( + isNegative, bytes, 0, bytes.length, scale)) { + return null; + } + if (isNegative) { + result.fastNegate(); + } + return result; + } + + @HiveDecimalVersionV2 + public static HiveDecimal create( + boolean isNegative, byte[] bytes, int offset, int length, int scale) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromDigitsOnlyBytesAndScale( + isNegative, bytes, offset, length, scale)) { + return null; + } + return result; + } + + /** + * Create a HiveDecimal by parsing the characters in a slice of a byte array. + * + * Same rules as create(String string) above. + * + */ + @HiveDecimalVersionV2 + public static HiveDecimal create(byte[] bytes, int offset, int length) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBytes( + bytes, offset, length, false)) { + return null; + } + return result; + } + + /** + * Same as the method above, except blanks before and after are tolerated. + * + */ + @HiveDecimalVersionV2 + public static HiveDecimal create( + byte[] bytes, int offset, int length, boolean trimBlanks) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBytes( + bytes, offset, length, trimBlanks)) { + return null; + } + return result; + } + + /** + * Create a HiveDecimal object from an int. + * + */ + @HiveDecimalVersionV1 + public static HiveDecimal create(int intValue) { + HiveDecimal result = new HiveDecimal(); + result.fastSetFromInt(intValue); + return result; + } + + /** + * Create a HiveDecimal object from a long. + * + */ + @HiveDecimalVersionV1 + public static HiveDecimal create(long longValue) { + HiveDecimal result = new HiveDecimal(); + result.fastSetFromLong(longValue); + return result; + } + + /** + * Create a HiveDecimal object from a long with a specified scale. + * + */ + @HiveDecimalVersionV2 + public static HiveDecimal create(long longValue, int scale) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromLongAndScale( + longValue, scale)) { + return null; + } + return result; + } + + /** + * Create a HiveDecimal object from a float. + *

+ * This method is equivalent to HiveDecimal.create(Float.toString(floatValue)) + */ + @HiveDecimalVersionV2 + public static HiveDecimal create(float floatValue) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromFloat(floatValue)) { + return null; + } + return result; + } + + /** + * Create a HiveDecimal object from a double. + *

+ * This method is equivalent to HiveDecimal.create(Double.toString(doubleValue)) + */ + @HiveDecimalVersionV2 + public static HiveDecimal create(double doubleValue) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromDouble(doubleValue)) { + return null; + } + return result; + } + + //----------------------------------------------------------------------------------------------- + // Serialization methods. + //----------------------------------------------------------------------------------------------- + + // The byte length of the scratch byte array that needs to be passed to serializationUtilsRead. + @HiveDecimalVersionV2 + public static final int SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ = + FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ; + + /** + * Deserialize data written in the format used by the SerializationUtils methods + * readBigInteger/writeBigInteger and create a decimal using the supplied scale. + *

+ * ORC uses those SerializationUtils methods for its serialization. + *

+ * A scratch bytes array is necessary to do the binary to decimal conversion for better + * performance. Pass a SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ byte array for scratchBytes. + *

+ * @return The deserialized decimal or null if the conversion failed. + */ + @HiveDecimalVersionV2 + public static HiveDecimal serializationUtilsRead( + InputStream inputStream, int scale, + byte[] scratchBytes) + throws IOException { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSerializationUtilsRead( + inputStream, scale, + scratchBytes)) { + return null; + } + return result; + } + + /** + * Convert bytes in the format used by BigInteger's toByteArray format (and accepted by its + * constructor) into a decimal using the specified scale. + *

+ * Our bigIntegerBytes methods create bytes in this format, too. + *

+ * This method is designed for high performance and does not create an actual BigInteger during + * binary to decimal conversion. + * + */ + @HiveDecimalVersionV2 + public static HiveDecimal createFromBigIntegerBytesAndScale( + byte[] bytes, int scale) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBigIntegerBytesAndScale( + bytes, 0, bytes.length, scale)) { + return null; + } + return result; + } + + @HiveDecimalVersionV2 + public static HiveDecimal createFromBigIntegerBytesAndScale( + byte[] bytes, int offset, int length, int scale) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBigIntegerBytesAndScale( + bytes, offset, length, scale)) { + return null; + } + return result; + } + + // The length of the long array that needs to be passed to serializationUtilsWrite. + @HiveDecimalVersionV2 + public static final int SCRATCH_LONGS_LEN = FAST_SCRATCH_LONGS_LEN; + + /** + * Serialize this decimal's BigInteger equivalent unscaled value using the format that the + * SerializationUtils methods readBigInteger/writeBigInteger use. + *

+ * ORC uses those SerializationUtils methods for its serialization. + *

+ * Scratch objects necessary to do the decimal to binary conversion without actually creating a + * BigInteger object are passed for better performance. + *

+ * Allocate scratchLongs with SCRATCH_LONGS_LEN longs. + * + */ + @HiveDecimalVersionV2 + public boolean serializationUtilsWrite( + OutputStream outputStream, + long[] scratchLongs) + throws IOException { + return + fastSerializationUtilsWrite( + outputStream, + scratchLongs); + } + + // The length of the scratch byte array that needs to be passed to bigIntegerBytes, etc. + @HiveDecimalVersionV2 + public static final int SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES = + FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES; + + /** + * Return binary representation of this decimal's BigInteger equivalent unscaled value using + * the format that the BigInteger's toByteArray method returns (and the BigInteger constructor + * accepts). + *

+ * Used by LazyBinary, Avro, and Parquet serialization. + *

+ * Scratch objects necessary to do the decimal to binary conversion without actually creating a + * BigInteger object are passed for better performance. + *

+ * Allocate scratchLongs with SCRATCH_LONGS_LEN longs. + * And, allocate buffer with SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes. + *

+ * @param scratchLongs + * @param buffer + * @return The number of bytes used for the binary result in buffer. Otherwise, 0 if the + * conversion failed. + */ + @HiveDecimalVersionV2 + public int bigIntegerBytes( + long[] scratchLongs, byte[] buffer) { + return + fastBigIntegerBytes( + scratchLongs, buffer); + } + + @HiveDecimalVersionV2 + public byte[] bigIntegerBytes() { + long[] scratchLongs = new long[SCRATCH_LONGS_LEN]; + byte[] buffer = new byte[SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; + final int byteLength = + fastBigIntegerBytes( + scratchLongs, buffer); + return Arrays.copyOfRange(buffer, 0, byteLength); + } + + /** + * Convert decimal to BigInteger binary bytes with a serialize scale, similar to the formatScale + * for toFormatString. It adds trailing zeroes the (emulated) BigInteger toByteArray result + * when a serializeScale is greater than current scale. Or, rounds if scale is less than + * current scale. + *

+ * Used by Avro and Parquet serialization. + *

+ * This emulates the OldHiveDecimal setScale AND THEN OldHiveDecimal getInternalStorage() behavior. + * + */ + @HiveDecimalVersionV2 + public int bigIntegerBytesScaled( + int serializeScale, + long[] scratchLongs, byte[] buffer) { + return + fastBigIntegerBytesScaled( + serializeScale, + scratchLongs, buffer); + } + + @HiveDecimalVersionV2 + public byte[] bigIntegerBytesScaled(int serializeScale) { + long[] scratchLongs = new long[SCRATCH_LONGS_LEN]; + byte[] buffer = new byte[SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; + int byteLength = + fastBigIntegerBytesScaled( + serializeScale, + scratchLongs, buffer); + return Arrays.copyOfRange(buffer, 0, byteLength); + } + + //----------------------------------------------------------------------------------------------- + // Convert to string/UTF-8 ASCII bytes methods. + //----------------------------------------------------------------------------------------------- + + /** + * Return a string representation of the decimal. + *

+ * It is the equivalent of calling bigDecimalValue().toPlainString -- it does not add exponent + * notation -- but is much faster. + *

+ * NOTE: If setScale(int serializationScale) was used to create the decimal object, then trailing + * fractional digits will be added to display to the serializationScale. Or, the display may + * get rounded. See the comments for that method. + * + */ + @HiveDecimalVersionV1 + @Override + public String toString() { + if (fastSerializationScale() != -1) { + + // Use the serialization scale and format the string with trailing zeroes (or + // round the decimal) if necessary. + return + fastToFormatString(fastSerializationScale()); + } else { + return + fastToString(); + } + } + + @HiveDecimalVersionV2 + public String toString( + byte[] scratchBuffer) { + if (fastSerializationScale() != -1) { + + // Use the serialization scale and format the string with trailing zeroes (or + // round the decimal) if necessary. + return + fastToFormatString( + fastSerializationScale(), + scratchBuffer); + } else { + return + fastToString(scratchBuffer); + } + } + + /** + * Return a string representation of the decimal using the specified scale. + *

+ * This method is designed to ALWAYS SUCCEED (unless the newScale parameter is out of range). + *

+ * Is does the equivalent of a setScale(int newScale). So, more than 38 digits may be returned. + * See that method for more details on how this can happen. + *

+ * @param formatScale The number of digits after the decimal point + * @return The scaled decimal representation string representation. + */ + @HiveDecimalVersionV1 + public String toFormatString(int formatScale) { + return + fastToFormatString( + formatScale); + } + + @HiveDecimalVersionV2 + public String toFormatString(int formatScale, byte[] scratchBuffer) { + return + fastToFormatString( + formatScale, + scratchBuffer); + } + + @HiveDecimalVersionV2 + public String toDigitsOnlyString() { + return + fastToDigitsOnlyString(); + } + + // The length of the scratch buffer that needs to be passed to toBytes, toFormatBytes, + // toDigitsOnlyBytes. + @HiveDecimalVersionV2 + public final static int SCRATCH_BUFFER_LEN_TO_BYTES = FAST_SCRATCH_BUFFER_LEN_TO_BYTES; + + /** + * Decimal to ASCII bytes conversion. + *

+ * The scratch buffer will contain the result afterwards. It should be + * SCRATCH_BUFFER_LEN_TO_BYTES bytes long. + *

+ * The result is produced at the end of the scratch buffer, so the return value is the byte + * index of the first byte. The byte slice is [byteIndex:SCRATCH_BUFFER_LEN_TO_BYTES-1]. + * + */ + @HiveDecimalVersionV2 + public int toBytes( + byte[] scratchBuffer) { + return + fastToBytes( + scratchBuffer); + } + + /** + * This is the serialization version of decimal to string conversion. + *

+ * It adds trailing zeroes when the formatScale is greater than the current scale. Or, it + * does round if the formatScale is less than the current scale. + *

+ * Note that you can get more than 38 (MAX_PRECISION) digits in the output with this method. + * + */ + @HiveDecimalVersionV2 + public int toFormatBytes( + int formatScale, + byte[] scratchBuffer) { + return + fastToFormatBytes( + formatScale, + scratchBuffer); + } + + /** + * Convert decimal to just the digits -- no dot. + *

+ * Currently used by BinarySortable serialization. + *

+ * A faster way to get just the digits than calling unscaledValue.toString().getBytes(). + * + */ + @HiveDecimalVersionV2 + public int toDigitsOnlyBytes( + byte[] scratchBuffer) { + return + fastToDigitsOnlyBytes( + scratchBuffer); + } + + //----------------------------------------------------------------------------------------------- + // Comparison methods. + //----------------------------------------------------------------------------------------------- + + @HiveDecimalVersionV1 + @Override + public int compareTo(HiveDecimal dec) { + return fastCompareTo(dec); + } + + /** + * Hash code based on (new) decimal representation. + *

+ * Faster than hashCode(). + *

+ * Used by map join and other Hive internal purposes where performance is important. + *

+ * IMPORTANT: See comments for hashCode(), too. + */ + @HiveDecimalVersionV2 + public int newFasterHashCode() { + return fastNewFasterHashCode(); + } + + /** + * This is returns original hash code as returned by HiveDecimalV1. + *

+ * We need this when the HiveDecimalV1 hash code has been exposed and and written or affected + * how data is written. + *

+ * This method supports compatibility. + *

+ * Examples: bucketing, Hive hash() function, and Hive statistics. + *

+ * NOTE: It is necessary to create a BigDecimal object and use its hash code, so this method is + * slow. + */ + @HiveDecimalVersionV1 + @Override + public int hashCode() { + return fastHashCode(); + } + + /** + * Are two decimal content (values) equal? + *

+ * @param obj The 2nd decimal. + * @return When obj is null or not class HiveDecimal, the return is false. + * Otherwise, returns true when the decimal values are exactly equal. + */ + @HiveDecimalVersionV1 + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + return fastEquals((HiveDecimal) obj); + } + + + //----------------------------------------------------------------------------------------------- + // Attribute methods. + //----------------------------------------------------------------------------------------------- + + /** + * Returns the scale of the decimal. Range 0 .. MAX_SCALE. + * + */ + @HiveDecimalVersionV1 + public int scale() { + return fastScale(); + } + + /** + * Returns the number of integer digits in the decimal. + *

+ * When the integer portion is zero, this method returns 0. + * + */ + @HiveDecimalVersionV2 + public int integerDigitCount() { + return fastIntegerDigitCount(); + } + + /** + * Returns the number of digits (integer and fractional) in the number, which is equivalent + * to SQL decimal precision. + *

+ * Note that this method is different from rawPrecision(), which returns the number of digits + * ignoring the scale. Note that rawPrecision returns 0 when the value is 0. + * + * Decimal precision rawPrecision + * 0 1 0 + * 1 1 1 + * -7 1 1 + * 0.1 1 1 + * 0.04 2 1 + * 0.00380 5 3 + * 104.0009 7 7 + *

+ * If you just want the actual number of digits, use rawPrecision(). + * + */ + @HiveDecimalVersionV1 + public int precision() { + return fastSqlPrecision(); + } + + // See comments for sqlPrecision. + @HiveDecimalVersionV2 + public int rawPrecision() { + return fastRawPrecision(); + } + + /** + * Get the sign of the decimal. + *

+ * @return 0 if the decimal is equal to 0, -1 if less than zero, and 1 if greater than 0 + */ + @HiveDecimalVersionV1 + public int signum() { + return fastSignum(); + } + + //----------------------------------------------------------------------------------------------- + // Value conversion methods. + //----------------------------------------------------------------------------------------------- + + /** + * Is the decimal value a byte? Range -128 to 127. + * Byte.MIN_VALUE Byte.MAX_VALUE + *

+ * Emulates testing for no value corruption: + * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().byteValue())) + *

+ * NOTE: Fractional digits are ignored in the test since byteValue() will + * remove them (round down). + *

+ * @return True when byteValue() will return a correct byte. + */ + @HiveDecimalVersionV2 + public boolean isByte() { + return fastIsByte(); + } + + /** + * A byte variation of longValue() + *

+ * This method will return a corrupted value unless isByte() is true. + */ + @HiveDecimalVersionV1 + public byte byteValue() { + return fastByteValueClip(); + } + + /** + * Is the decimal value a short? Range -32,768 to 32,767. + * Short.MIN_VALUE Short.MAX_VALUE + *

+ * Emulates testing for no value corruption: + * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().shortValue())) + *

+ * NOTE: Fractional digits are ignored in the test since shortValue() will + * remove them (round down). + *

+ * @return True when shortValue() will return a correct short. + */ + @HiveDecimalVersionV2 + public boolean isShort() { + return fastIsShort(); + } + + /** + * A short variation of longValue(). + *

+ * This method will return a corrupted value unless isShort() is true. + */ + @HiveDecimalVersionV1 + public short shortValue() { + return fastShortValueClip(); + } + + /** + * Is the decimal value a int? Range -2,147,483,648 to 2,147,483,647. + * Integer.MIN_VALUE Integer.MAX_VALUE + *

+ * Emulates testing for no value corruption: + * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().intValue())) + *

+ * NOTE: Fractional digits are ignored in the test since intValue() will + * remove them (round down). + *

+ * @return True when intValue() will return a correct int. + */ + @HiveDecimalVersionV2 + public boolean isInt() { + return fastIsInt(); + } + + /** + * An int variation of longValue(). + *

+ * This method will return a corrupted value unless isInt() is true. + */ + @HiveDecimalVersionV1 + public int intValue() { + return fastIntValueClip(); + } + + /** + * Is the decimal value a long? Range -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807. + * Long.MIN_VALUE Long.MAX_VALUE + *

+ * Emulates testing for no value corruption: + * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().longValue())) + *

+ * NOTE: Fractional digits are ignored in the test since longValue() will + * remove them (round down). + *

+ * @return True when longValue() will return a correct long. + */ + @HiveDecimalVersionV2 + public boolean isLong() { + return fastIsLong(); + } + + /** + * Return the long value of a decimal. + *

+ * This method will return a corrupted value unless isLong() is true. + */ + @HiveDecimalVersionV1 + public long longValue() { + return fastLongValueClip(); + } + + @HiveDecimalVersionV1 + public long longValueExact() { + if (!isLong()) { + throw new ArithmeticException(); + } + return fastLongValueClip(); + } + + /** + * Return a float representing the decimal. Due the limitations of float, some values will not + * be accurate. + * + */ + @HiveDecimalVersionV1 + public float floatValue() { + return fastFloatValue(); + } + + /** + * Return a double representing the decimal. Due the limitations of double, some values will not + * be accurate. + * + */ + @HiveDecimalVersionV1 + public double doubleValue() { + return fastDoubleValue(); + } + + /** + * Return a BigDecimal representing the decimal. The BigDecimal class is able to accurately + * represent the decimal. + * + * NOTE: We are not representing our decimal as BigDecimal now as OldHiveDecimal did, so this + * is now slower. + * + */ + @HiveDecimalVersionV1 + public BigDecimal bigDecimalValue() { + return fastBigDecimalValue(); + } + + /** + * Get a BigInteger representing the decimal's digits without a dot. + *

+ * @return Returns a signed BigInteger. + */ + @HiveDecimalVersionV1 + public BigInteger unscaledValue() { + return fastBigIntegerValue(); + } + + /** + * Return a decimal with only the fractional digits. + *

+ * Zero is returned when there are no fractional digits (i.e. scale is 0). + * + */ + @HiveDecimalVersionV2 + public HiveDecimal fractionPortion() { + HiveDecimal result = new HiveDecimal(); + result.fastFractionPortion(); + return result; + } + + /** + * Return a decimal with only the integer digits. + *

+ * Any fractional digits are removed. E.g. 2.083 scale 3 returns as 2 scale 0. + * + */ + @HiveDecimalVersionV2 + public HiveDecimal integerPortion() { + HiveDecimal result = new HiveDecimal(); + result.fastIntegerPortion(); + return result; + } + + //----------------------------------------------------------------------------------------------- + // Math methods. + //----------------------------------------------------------------------------------------------- + + /** + * Add the current decimal and another decimal and return the result. + * + */ + @HiveDecimalVersionV1 + public HiveDecimal add(HiveDecimal dec) { + HiveDecimal result = new HiveDecimal(); + if (!fastAdd( + dec, + result)) { + return null; + } + return result; + } + + /** + * Subtract from the current decimal another decimal and return the result. + * + */ + @HiveDecimalVersionV1 + public HiveDecimal subtract(HiveDecimal dec) { + HiveDecimal result = new HiveDecimal(); + if (!fastSubtract( + dec, + result)) { + return null; + } + return result; + } + + /** + * Multiply two decimals. + *

+ * NOTE: Overflow Determination for Multiply + *

+ * OldDecimal.multiply performs the multiply with BigDecimal but DOES NOT ALLOW ROUNDING + * (i.e. no throwing away lower fractional digits). + *

+ * CONSIDER: Allowing rounding. This would eliminate cases today where we return null for + * the multiplication result. + *

+ * IMPLEMENTATION NOTE: HiveDecimalV1 code does this: + *

+ * return create(bd.multiply(dec.bd), false); + */ + @HiveDecimalVersionV1 + public HiveDecimal multiply(HiveDecimal dec) { + HiveDecimal result = new HiveDecimal(); + if (!fastMultiply( + dec, + result)) { + return null; + } + return result; + } + + /** + * Multiplies a decimal by a power of 10. + *

+ * The decimal 19350 scale 0 will return 193.5 scale 1 when power is -2 (negative). + *

+ * The decimal 1.000923 scale 6 will return 10009.23 scale 2 when power is 4 (positive). + *

+ * @param power + * @return Returns a HiveDecimal whose value is value * 10^power. + */ + @HiveDecimalVersionV1 + public HiveDecimal scaleByPowerOfTen(int power) { + if (power == 0 || fastSignum() == 0) { + // No change for multiply by 10^0 or value 0. + return this; + } + HiveDecimal result = new HiveDecimal(); + if (!fastScaleByPowerOfTen( + power, + result)) { + return null; + } + return result; + } + + /** + * Take the absolute value of a decimal. + *

+ * @return When the decimal is negative, returns a new HiveDecimal with the positive value. + * Otherwise, returns the current 0 or positive value object; + */ + @HiveDecimalVersionV1 + public HiveDecimal abs() { + if (fastSignum() != -1) { + return this; + } + HiveDecimal result = new HiveDecimal(this); + result.fastAbs(); + return result; + } + + /** + * Reverse the sign of a decimal. + *

+ * @return Returns a new decimal with the sign flipped. When the value is 0, the current + * object is returned. + */ + @HiveDecimalVersionV1 + public HiveDecimal negate() { + if (fastSignum() == 0) { + return this; + } + HiveDecimal result = new HiveDecimal(this); + result.fastNegate(); + return result; + } + + //----------------------------------------------------------------------------------------------- + // Rounding / setScale methods. + //----------------------------------------------------------------------------------------------- + + /** + * DEPRECATED for V2. + *

+ * Create a decimal from another decimal whose only change is it is MARKED and will display / + * serialize with a specified scale that will add trailing zeroes (or round) if necessary. + *

+ * After display / serialization, the MARKED object is typically thrown away. + *

+ * A MARKED decimal ONLY affects these 2 methods since these were the only ways setScale was + * used in the old code. + *

+ * toString + * unscaleValue + *

+ * This method has been deprecated because has poor performance by creating a throw away object. + *

+ * For setScale(scale).toString() use toFormatString(scale) instead. + * For setScale(scale).unscaledValue().toByteArray() use V2 bigIntegerBytesScaled(scale) instead. + *

+ * For better performance, use the V2 form of toFormatString that takes a scratch buffer, + * or even better use toFormatBytes. + *

+ * And, use the form of bigIntegerBytesScaled that takes scratch objects for better performance. + * + */ + @Deprecated + @HiveDecimalVersionV1 + public HiveDecimal setScale(int serializationScale) { + HiveDecimal result = new HiveDecimal(this); + result.fastSetSerializationScale(serializationScale); + return result; + } + + /** + * Do decimal rounding and return the result. + *

+ * When the roundingPoint is 0 or positive, we round away lower fractional digits if the + * roundingPoint is less than current scale. In this case, we will round the result using the + * specified rounding mode. + *

+ * When the roundingPoint is negative, the rounding will occur within the integer digits. Integer + * digits below the roundPoint will be cleared. If the rounding occurred, a one will be added + * just above the roundingPoint. Note this may cause overflow. + *

+ * No effect when the roundingPoint equals the current scale. The current object is returned. + *

+ * The name setScale is taken from BigDecimal.setScale -- a better name would have been round. + * + */ + @HiveDecimalVersionV1 + public HiveDecimal setScale( + int roundingPoint, int roundingMode) { + if (fastScale() == roundingPoint) { + // No change. + return this; + } + + // Even if we are just setting the scale when newScale is greater than the current scale, + // we need a new object to obey our immutable behavior. + HiveDecimal result = new HiveDecimal(); + if (!fastRound( + roundingPoint, roundingMode, + result)) { + return null; + } + return result; + } + + /** + * Return the result of decimal^exponent + *

+ * CONSIDER: Currently, negative exponent is not supported. + * CONSIDER: Does anybody use this method? + * + */ + @HiveDecimalVersionV1 + public HiveDecimal pow(int exponent) { + HiveDecimal result = new HiveDecimal(this); + if (!fastPow( + exponent, result)) { + return null; + } + return result; + } + + /** + * Divides this decimal by another decimal and returns a new decimal with the result. + * + */ + @HiveDecimalVersionV1 + public HiveDecimal divide(HiveDecimal divisor) { + HiveDecimal result = new HiveDecimal(); + if (!fastDivide( + divisor, + result)) { + return null; + } + return result; + } + + /** + * Divides this decimal by another decimal and returns a new decimal with the remainder of the + * division. + *

+ * value is (decimal % divisor) + *

+ * The remainder is equivalent to BigDecimal: + * bigDecimalValue().subtract(bigDecimalValue().divideToIntegralValue(divisor).multiply(divisor)) + * + */ + @HiveDecimalVersionV1 + public HiveDecimal remainder(HiveDecimal divisor) { + HiveDecimal result = new HiveDecimal(); + if (!fastRemainder( + divisor, + result)) { + return null; + } + return result; + } + + //----------------------------------------------------------------------------------------------- + // Precision/scale enforcement methods. + //----------------------------------------------------------------------------------------------- + + /** + * Determine if a decimal fits within a specified maxPrecision and maxScale, and round + * off fractional digits if necessary to make the decimal fit. + *

+ * The relationship between the enforcement maxPrecision and maxScale is restricted. The + * specified maxScale must be less than or equal to the maxPrecision. + *

+ * Normally, decimals that result from creation operation, arithmetic operations, etc are + * "free range" up to MAX_PRECISION and MAX_SCALE. Each operation checks if the result decimal + * is beyond MAX_PRECISION and MAX_SCALE. If so the result decimal is rounded off using + * ROUND_HALF_UP. If the round digit is 5 or more, one is added to the lowest remaining digit. + * The round digit is the digit just below the round point. Result overflow can occur if a + * result decimal's integer portion exceeds MAX_PRECISION. + *

+ * This method supports enforcing to a declared Hive DECIMAL's precision/scale. + * E.g. DECIMAL(10,4) + *

+ * Here are the enforcement/rounding checks of this method: + *

+ * 1) Maximum integer digits = maxPrecision - maxScale + *

+ * If the decimal's integer digit count exceeds this, the decimal does not fit (overflow). + *

+ * 2) If decimal's scale is greater than maxScale, then excess fractional digits are + * rounded off. When rounding increases the remaining decimal, it may exceed the + * limits and overflow. + *

+ * @param dec + * @param maxPrecision + * @param maxScale + * @return The original decimal if no adjustment is necessary. + * A rounded off decimal if adjustment was necessary. + * Otherwise, null if the decimal doesn't fit within maxPrecision / maxScale or rounding + * caused a result that exceeds the specified limits or MAX_PRECISION integer digits. + */ + @HiveDecimalVersionV1 + public static HiveDecimal enforcePrecisionScale( + HiveDecimal dec, int maxPrecision, int maxScale) { + + if (maxPrecision < 1 || maxPrecision > MAX_PRECISION) { + throw new IllegalArgumentException(STRING_ENFORCE_PRECISION_OUT_OF_RANGE); + } + + if (maxScale < 0 || maxScale > HiveDecimal.MAX_SCALE) { + throw new IllegalArgumentException(STRING_ENFORCE_SCALE_OUT_OF_RANGE); + } + + if (maxPrecision < maxScale) { + throw new IllegalArgumentException(STRING_ENFORCE_SCALE_LESS_THAN_EQUAL_PRECISION); + } + + if (dec == null) { + return null; + } + + FastCheckPrecisionScaleStatus status = + dec.fastCheckPrecisionScale( + maxPrecision, maxScale); + switch (status) { + case NO_CHANGE: + return dec; + case OVERFLOW: + return null; + case UPDATE_SCALE_DOWN: + { + HiveDecimal result = new HiveDecimal(); + if (!dec.fastUpdatePrecisionScale( + maxPrecision, maxScale, status, + result)) { + return null; + } + return result; + } + default: + throw new RuntimeException("Unknown fast decimal check precision and scale status " + status); + } + } + + //----------------------------------------------------------------------------------------------- + // Validation methods. + //----------------------------------------------------------------------------------------------- + + /** + * Throws an exception if the current decimal value is invalid. + */ + @HiveDecimalVersionV2 + public void validate() { + if (!fastIsValid()) { + fastRaiseInvalidException(); + } + } +} \ No newline at end of file diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalV1.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalV1.java new file mode 100644 index 0000000000000000000000000000000000000000..f99ffee32763ea3e61859e4c52cda9af85ed73a2 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalV1.java @@ -0,0 +1,386 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; + +/** + * + * HiveDecimal. Simple wrapper for BigDecimal. Adds fixed max precision and non scientific string + * representation + * + */ +public final class HiveDecimalV1 implements Comparable { + @HiveDecimalVersionV1 + public static final int MAX_PRECISION = 38; + @HiveDecimalVersionV1 + public static final int MAX_SCALE = 38; + + /** + * Default precision/scale when user doesn't specify in the column metadata, such as + * decimal and decimal(8). + */ + @HiveDecimalVersionV1 + public static final int USER_DEFAULT_PRECISION = 10; + @HiveDecimalVersionV1 + public static final int USER_DEFAULT_SCALE = 0; + + /** + * Default precision/scale when system is not able to determine them, such as in case + * of a non-generic udf. + */ + @HiveDecimalVersionV1 + public static final int SYSTEM_DEFAULT_PRECISION = 38; + @HiveDecimalVersionV1 + public static final int SYSTEM_DEFAULT_SCALE = 18; + + @HiveDecimalVersionV1 + public static final HiveDecimalV1 ZERO = new HiveDecimalV1(BigDecimal.ZERO); + @HiveDecimalVersionV1 + public static final HiveDecimalV1 ONE = new HiveDecimalV1(BigDecimal.ONE); + + @HiveDecimalVersionV1 + public static final int ROUND_FLOOR = BigDecimal.ROUND_FLOOR; + @HiveDecimalVersionV1 + public static final int ROUND_CEILING = BigDecimal.ROUND_CEILING; + @HiveDecimalVersionV1 + public static final int ROUND_HALF_UP = BigDecimal.ROUND_HALF_UP; + @HiveDecimalVersionV1 + public static final int ROUND_HALF_EVEN = BigDecimal.ROUND_HALF_EVEN; + + private BigDecimal bd = BigDecimal.ZERO; + + private HiveDecimalV1(BigDecimal bd) { + this.bd = bd; + } + + @HiveDecimalVersionV1 + public static HiveDecimalV1 create(BigDecimal b) { + return create(b, true); + } + + @HiveDecimalVersionV1 + public static HiveDecimalV1 create(BigDecimal b, boolean allowRounding) { + BigDecimal bd = normalize(b, allowRounding); + return bd == null ? null : new HiveDecimalV1(bd); + } + + @HiveDecimalVersionV1 + public static HiveDecimalV1 create(BigInteger unscaled, int scale) { + BigDecimal bd = normalize(new BigDecimal(unscaled, scale), true); + return bd == null ? null : new HiveDecimalV1(bd); + } + + @HiveDecimalVersionV1 + public static HiveDecimalV1 create(String dec) { + BigDecimal bd; + try { + bd = new BigDecimal(dec.trim()); + } catch (NumberFormatException ex) { + return null; + } + bd = normalize(bd, true); + return bd == null ? null : new HiveDecimalV1(bd); + } + + @HiveDecimalVersionV1 + public static HiveDecimalV1 create(BigInteger bi) { + BigDecimal bd = normalize(new BigDecimal(bi), true); + return bd == null ? null : new HiveDecimalV1(bd); + } + + @HiveDecimalVersionV1 + public static HiveDecimalV1 create(int i) { + return new HiveDecimalV1(new BigDecimal(i)); + } + + @HiveDecimalVersionV1 + public static HiveDecimalV1 create(long l) { + return new HiveDecimalV1(new BigDecimal(l)); + } + + @HiveDecimalVersionV1 + @Override + public String toString() { + return bd.toPlainString(); + } + + /** + * Return a string representation of the number with the number of decimal digits as + * the given scale. Please note that this is different from toString(). + * @param scale the number of digits after the decimal point + * @return the string representation of exact number of decimal digits + */ + @HiveDecimalVersionV1 + public String toFormatString(int scale) { + return (bd.scale() == scale ? bd : + bd.setScale(scale, RoundingMode.HALF_UP)).toPlainString(); + } + + @HiveDecimalVersionV1 + public HiveDecimalV1 setScale(int i) { + return new HiveDecimalV1(bd.setScale(i, RoundingMode.HALF_UP)); + } + + @HiveDecimalVersionV1 + @Override + public int compareTo(HiveDecimalV1 dec) { + return bd.compareTo(dec.bd); + } + + @HiveDecimalVersionV1 + @Override + public int hashCode() { + return bd.hashCode(); + } + + @HiveDecimalVersionV1 + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + return bd.equals(((HiveDecimalV1) obj).bd); + } + + @HiveDecimalVersionV1 + public int scale() { + return bd.scale(); + } + + /** + * Returns the number of digits (integer and fractional) in the number, which is equivalent + * to SQL decimal precision. Note that this is different from BigDecimal.precision(), + * which returns the precision of the unscaled value (BigDecimal.valueOf(0.01).precision() = 1, + * whereas HiveDecimal.create("0.01").precision() = 2). + * If you want the BigDecimal precision, use HiveDecimal.bigDecimalValue().precision() + * @return + */ + @HiveDecimalVersionV1 + public int precision() { + int bdPrecision = bd.precision(); + int bdScale = bd.scale(); + + if (bdPrecision < bdScale) { + // This can happen for numbers less than 0.1 + // For 0.001234: bdPrecision=4, bdScale=6 + // In this case, we'll set the type to have the same precision as the scale. + return bdScale; + } + return bdPrecision; + } + + /** Note - this method will corrupt the value if it doesn't fit. */ + @HiveDecimalVersionV1 + public int intValue() { + return bd.intValue(); + } + + @HiveDecimalVersionV1 + public double doubleValue() { + return bd.doubleValue(); + } + + /** Note - this method will corrupt the value if it doesn't fit. */ + @HiveDecimalVersionV1 + public long longValue() { + return bd.longValue(); + } + + /** Note - this method will corrupt the value if it doesn't fit. */ + @HiveDecimalVersionV1 + public short shortValue() { + return bd.shortValue(); + } + + @HiveDecimalVersionV1 + public float floatValue() { + return bd.floatValue(); + } + + @HiveDecimalVersionV1 + public BigDecimal bigDecimalValue() { + return bd; + } + + @HiveDecimalVersionV1 + public byte byteValue() { + return bd.byteValue(); + } + + @HiveDecimalVersionV1 + public HiveDecimalV1 setScale(int adjustedScale, int rm) { + return create(bd.setScale(adjustedScale, rm)); + } + + @HiveDecimalVersionV1 + public HiveDecimalV1 subtract(HiveDecimalV1 dec) { + return create(bd.subtract(dec.bd)); + } + + @HiveDecimalVersionV1 + public HiveDecimalV1 multiply(HiveDecimalV1 dec) { + return create(bd.multiply(dec.bd), false); + } + + @HiveDecimalVersionV1 + public BigInteger unscaledValue() { + return bd.unscaledValue(); + } + + @HiveDecimalVersionV1 + public HiveDecimalV1 scaleByPowerOfTen(int n) { + return create(bd.scaleByPowerOfTen(n)); + } + + @HiveDecimalVersionV1 + public HiveDecimalV1 abs() { + return create(bd.abs()); + } + + @HiveDecimalVersionV1 + public HiveDecimalV1 negate() { + return create(bd.negate()); + } + + @HiveDecimalVersionV1 + public HiveDecimalV1 add(HiveDecimalV1 dec) { + return create(bd.add(dec.bd)); + } + + @HiveDecimalVersionV1 + public HiveDecimalV1 pow(int n) { + BigDecimal result = normalize(bd.pow(n), false); + return result == null ? null : new HiveDecimalV1(result); + } + + @HiveDecimalVersionV1 + public HiveDecimalV1 remainder(HiveDecimalV1 dec) { + return create(bd.remainder(dec.bd)); + } + + @HiveDecimalVersionV1 + public HiveDecimalV1 divide(HiveDecimalV1 dec) { + return create(bd.divide(dec.bd, MAX_SCALE, RoundingMode.HALF_UP), true); + } + + /** + * Get the sign of the underlying decimal. + * @return 0 if the decimal is equal to 0, -1 if less than zero, and 1 if greater than 0 + */ + @HiveDecimalVersionV1 + public int signum() { + return bd.signum(); + } + + private static BigDecimal trim(BigDecimal d) { + if (d.compareTo(BigDecimal.ZERO) == 0) { + // Special case for 0, because java doesn't strip zeros correctly on that number. + d = BigDecimal.ZERO; + } else { + d = d.stripTrailingZeros(); + if (d.scale() < 0) { + // no negative scale decimals + d = d.setScale(0); + } + } + return d; + } + + private static BigDecimal normalize(BigDecimal bd, boolean allowRounding) { + if (bd == null) { + return null; + } + + bd = trim(bd); + + int intDigits = bd.precision() - bd.scale(); + + if (intDigits > MAX_PRECISION) { + return null; + } + + int maxScale = Math.min(MAX_SCALE, Math.min(MAX_PRECISION - intDigits, bd.scale())); + if (bd.scale() > maxScale ) { + if (allowRounding) { + bd = bd.setScale(maxScale, RoundingMode.HALF_UP); + // Trimming is again necessary, because rounding may introduce new trailing 0's. + bd = trim(bd); + } else { + bd = null; + } + } + + return bd; + } + + private static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, int maxScale) { + if (bd == null) { + return null; + } + + /** + * Specially handling the case that bd=0, and we are converting it to a type where precision=scale, + * such as decimal(1, 1). + */ + if (bd.compareTo(BigDecimal.ZERO) == 0 && bd.scale() == 0 && maxPrecision == maxScale) { + return bd.setScale(maxScale); + } + + bd = trim(bd); + + if (bd.scale() > maxScale) { + bd = bd.setScale(maxScale, RoundingMode.HALF_UP); + } + + int maxIntDigits = maxPrecision - maxScale; + int intDigits = bd.precision() - bd.scale(); + if (intDigits > maxIntDigits) { + return null; + } + + return bd; + } + + @HiveDecimalVersionV1 + public static HiveDecimalV1 enforcePrecisionScale(HiveDecimalV1 dec, int maxPrecision, int maxScale) { + if (dec == null) { + return null; + } + + // Minor optimization, avoiding creating new objects. + if (dec.precision() - dec.scale() <= maxPrecision - maxScale && + dec.scale() <= maxScale) { + return dec; + } + + BigDecimal bd = enforcePrecisionScale(dec.bd, maxPrecision, maxScale); + if (bd == null) { + return null; + } + + return HiveDecimalV1.create(bd); + } + + @HiveDecimalVersionV1 + public long longValueExact() { + return bd.longValueExact(); + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV1.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV1.java new file mode 100644 index 0000000000000000000000000000000000000000..82b769a15eb9fa884f44894e8555b7c80dbdfc50 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV1.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.type; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Marks methods including static methods and fields as being part of version 1 HiveDecimal. + * + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +public @interface HiveDecimalVersionV1 { + +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV2.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV2.java new file mode 100644 index 0000000000000000000000000000000000000000..a47513ebacb9c16600e9e2e50c194d13c19d39c9 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV2.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.type; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Marks methods including static methods and fields as being part of version 2 HiveDecimal. + * + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +public @interface HiveDecimalVersionV2 { + +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java new file mode 100644 index 0000000000000000000000000000000000000000..cb1306ee78ca99c00ef1a2ab8fcf439029f58dd4 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java @@ -0,0 +1,251 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import java.math.BigDecimal; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.hive.common.util.IntervalDayTimeUtils; + + +/** + * Day-time interval type representing an offset in days/hours/minutes/seconds, + * with nanosecond precision. + * 1 day = 24 hours = 1440 minutes = 86400 seconds + */ +public class HiveIntervalDayTime implements Comparable { + + // days/hours/minutes/seconds all represented as seconds + protected long totalSeconds; + protected int nanos; + + public HiveIntervalDayTime() { + } + + public HiveIntervalDayTime(int days, int hours, int minutes, int seconds, int nanos) { + set(days, hours, minutes, seconds, nanos); + } + + public HiveIntervalDayTime(long seconds, int nanos) { + set(seconds, nanos); + } + + public HiveIntervalDayTime(BigDecimal seconds) { + set(seconds); + } + + public HiveIntervalDayTime(HiveIntervalDayTime other) { + set(other.totalSeconds, other.nanos); + } + + public int getDays() { + return (int) TimeUnit.SECONDS.toDays(totalSeconds); + } + + public int getHours() { + return (int) (TimeUnit.SECONDS.toHours(totalSeconds) % TimeUnit.DAYS.toHours(1)); + } + + public int getMinutes() { + return (int) (TimeUnit.SECONDS.toMinutes(totalSeconds) % TimeUnit.HOURS.toMinutes(1)); + } + + public int getSeconds() { + return (int) (totalSeconds % TimeUnit.MINUTES.toSeconds(1)); + } + + public int getNanos() { + return nanos; + } + + /** + * Returns days/hours/minutes all converted into seconds. + * Nanos still need to be retrieved using getNanos() + * @return + */ + public long getTotalSeconds() { + return totalSeconds; + } + + /** + * + * @return double representation of the interval day time, accurate to nanoseconds + */ + public double getDouble() { + return totalSeconds + nanos / 1000000000; + } + + /** + * Ensures that the seconds and nanoseconds fields have consistent sign + */ + protected void normalizeSecondsAndNanos() { + if (totalSeconds > 0 && nanos < 0) { + --totalSeconds; + nanos += IntervalDayTimeUtils.NANOS_PER_SEC; + } else if (totalSeconds < 0 && nanos > 0) { + ++totalSeconds; + nanos -= IntervalDayTimeUtils.NANOS_PER_SEC; + } + } + + public void set(int days, int hours, int minutes, int seconds, int nanos) { + long totalSeconds = seconds; + totalSeconds += TimeUnit.DAYS.toSeconds(days); + totalSeconds += TimeUnit.HOURS.toSeconds(hours); + totalSeconds += TimeUnit.MINUTES.toSeconds(minutes); + totalSeconds += TimeUnit.NANOSECONDS.toSeconds(nanos); + nanos = nanos % IntervalDayTimeUtils.NANOS_PER_SEC; + + this.totalSeconds = totalSeconds; + this.nanos = nanos; + + normalizeSecondsAndNanos(); + } + + public void set(long seconds, int nanos) { + this.totalSeconds = seconds; + this.nanos = nanos; + normalizeSecondsAndNanos(); + } + + public void set(BigDecimal totalSecondsBd) { + long totalSeconds = totalSecondsBd.longValue(); + BigDecimal fractionalSecs = totalSecondsBd.remainder(BigDecimal.ONE); + int nanos = fractionalSecs.multiply(IntervalDayTimeUtils.NANOS_PER_SEC_BD).intValue(); + set(totalSeconds, nanos); + } + + public void set(HiveIntervalDayTime other) { + set(other.getTotalSeconds(), other.getNanos()); + } + + public HiveIntervalDayTime negate() { + return new HiveIntervalDayTime(-getTotalSeconds(), -getNanos()); + } + + @Override + public int compareTo(HiveIntervalDayTime other) { + long cmp = this.totalSeconds - other.totalSeconds; + if (cmp == 0) { + cmp = this.nanos - other.nanos; + } + if (cmp != 0) { + cmp = cmp > 0 ? 1 : -1; + } + return (int) cmp; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof HiveIntervalDayTime)) { + return false; + } + return 0 == compareTo((HiveIntervalDayTime) obj); + } + + /** + * Return a copy of this object. + */ + @Override + public Object clone() { + return new HiveIntervalDayTime(totalSeconds, nanos); + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append(totalSeconds).append(nanos).toHashCode(); + } + + @Override + public String toString() { + // If normalize() was used, then day-hour-minute-second-nanos should have the same sign. + // This is currently working with that assumption. + boolean isNegative = (totalSeconds < 0 || nanos < 0); + String daySecondSignStr = isNegative ? "-" : ""; + + return String.format("%s%d %02d:%02d:%02d.%09d", + daySecondSignStr, Math.abs(getDays()), + Math.abs(getHours()), Math.abs(getMinutes()), + Math.abs(getSeconds()), Math.abs(getNanos())); + } + + public static HiveIntervalDayTime valueOf(String strVal) { + HiveIntervalDayTime result = null; + if (strVal == null) { + throw new IllegalArgumentException("Interval day-time string was null"); + } + Matcher patternMatcher = PATTERN_MATCHER.get(); + patternMatcher.reset(strVal); + if (patternMatcher.matches()) { + // Parse out the individual parts + try { + // Sign - whether interval is positive or negative + int sign = 1; + String field = patternMatcher.group(1); + if (field != null && field.equals("-")) { + sign = -1; + } + int days = sign * + IntervalDayTimeUtils.parseNumericValueWithRange("day", patternMatcher.group(2), + 0, Integer.MAX_VALUE); + byte hours = (byte) (sign * + IntervalDayTimeUtils.parseNumericValueWithRange("hour", patternMatcher.group(3), 0, 23)); + byte minutes = (byte) (sign * + IntervalDayTimeUtils.parseNumericValueWithRange("minute", patternMatcher.group(4), 0, 59)); + int seconds = 0; + int nanos = 0; + field = patternMatcher.group(5); + if (field != null) { + BigDecimal bdSeconds = new BigDecimal(field); + if (bdSeconds.compareTo(IntervalDayTimeUtils.MAX_INT_BD) > 0) { + throw new IllegalArgumentException("seconds value of " + bdSeconds + " too large"); + } + seconds = sign * bdSeconds.intValue(); + nanos = sign * bdSeconds.subtract(new BigDecimal(bdSeconds.toBigInteger())) + .multiply(IntervalDayTimeUtils.NANOS_PER_SEC_BD).intValue(); + } + + result = new HiveIntervalDayTime(days, hours, minutes, seconds, nanos); + } catch (Exception err) { + throw new IllegalArgumentException("Error parsing interval day-time string: " + strVal, err); + } + } else { + throw new IllegalArgumentException( + "Interval string does not match day-time format of 'd h:m:s.n': " + strVal); + } + + return result; + } + + // Simple pattern: D H:M:S.nnnnnnnnn + private final static String PARSE_PATTERN = + "([+|-])?(\\d+) (\\d+):(\\d+):((\\d+)(\\.(\\d+))?)"; + + private static final ThreadLocal PATTERN_MATCHER = new ThreadLocal() { + @Override + protected Matcher initialValue() { + return Pattern.compile(PARSE_PATTERN).matcher(""); + } + }; +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java new file mode 100644 index 0000000000000000000000000000000000000000..8ff95e343d59d12bf70052d33bd58a8f5c2d4ef2 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + + +/** + * + * HiveVarChar. + * String wrapper to support SQL VARCHAR features. + * Max string length is enforced. + * + */ +public class HiveVarchar extends HiveBaseChar + implements Comparable { + + public static final int MAX_VARCHAR_LENGTH = 65535; + + public HiveVarchar() { + } + + public HiveVarchar(String val, int len) { + setValue(val, len); + } + + public HiveVarchar(HiveVarchar hc, int len) { + setValue(hc, len); + } + + /** + * Set the new value + */ + public void setValue(String val) { + super.setValue(val, -1); + } + + public void setValue(HiveVarchar hc) { + super.setValue(hc.getValue(), -1); + } + + public int compareTo(HiveVarchar rhs) { + if (rhs == this) { + return 0; + } + return this.getValue().compareTo(rhs.getValue()); + } + + public boolean equals(Object rhs) { + if (rhs == this) { + return true; + } + return this.getValue().equals(((HiveVarchar)rhs).getValue()); + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..eeb3359b3f225ff0f701f6d8c6c8a6aae8ae522d --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java @@ -0,0 +1,187 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import java.sql.Date; +import java.sql.Timestamp; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +public class RandomTypeUtil { + + public static String getRandString(Random r) { + return getRandString(r, null, r.nextInt(10)); + } + + public static String getRandString(Random r, String characters, int length) { + if (characters == null) { + characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (characters == null) { + sb.append((char) (r.nextInt(128))); + } else { + sb.append(characters.charAt(r.nextInt(characters.length()))); + } + } + return sb.toString(); + } + + public static byte[] getRandBinary(Random r, int len){ + byte[] bytes = new byte[len]; + for (int j = 0; j < len; j++){ + bytes[j] = Byte.valueOf((byte) r.nextInt()); + } + return bytes; + } + + private static final String DECIMAL_CHARS = "0123456789"; + + public static HiveDecimal getRandHiveDecimal(Random r) { + int precision; + int scale; + while (true) { + StringBuilder sb = new StringBuilder(); + precision = 1 + r.nextInt(18); + scale = 0 + r.nextInt(precision + 1); + + int integerDigits = precision - scale; + + if (r.nextBoolean()) { + sb.append("-"); + } + + if (integerDigits == 0) { + sb.append("0"); + } else { + sb.append(getRandString(r, DECIMAL_CHARS, integerDigits)); + } + if (scale != 0) { + sb.append("."); + sb.append(getRandString(r, DECIMAL_CHARS, scale)); + } + + return HiveDecimal.create(sb.toString()); + } + } + + public static Date getRandDate(Random r) { + String dateStr = String.format("%d-%02d-%02d", + Integer.valueOf(1800 + r.nextInt(500)), // year + Integer.valueOf(1 + r.nextInt(12)), // month + Integer.valueOf(1 + r.nextInt(28))); // day + Date dateVal = Date.valueOf(dateStr); + return dateVal; + } + + /** + * TIMESTAMP. + */ + + public static final long NANOSECONDS_PER_SECOND = TimeUnit.SECONDS.toNanos(1); + public static final long MILLISECONDS_PER_SECOND = TimeUnit.SECONDS.toMillis(1); + public static final long NANOSECONDS_PER_MILLISSECOND = TimeUnit.MILLISECONDS.toNanos(1); + + private static final ThreadLocal DATE_FORMAT = + new ThreadLocal() { + @Override + protected DateFormat initialValue() { + return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + } + }; + + // We've switched to Joda/Java Calendar which has a more limited time range.... + public static final int MIN_YEAR = 1900; + public static final int MAX_YEAR = 3000; + private static final long MIN_FOUR_DIGIT_YEAR_MILLIS = parseToMillis("1900-01-01 00:00:00"); + private static final long MAX_FOUR_DIGIT_YEAR_MILLIS = parseToMillis("3000-01-01 00:00:00"); + + private static long parseToMillis(String s) { + try { + return DATE_FORMAT.get().parse(s).getTime(); + } catch (ParseException ex) { + throw new RuntimeException(ex); + } + } + + public static Timestamp getRandTimestamp(Random r) { + return getRandTimestamp(r, MIN_YEAR, MAX_YEAR); + } + + public static Timestamp getRandTimestamp(Random r, int minYear, int maxYear) { + String optionalNanos = ""; + switch (r.nextInt(4)) { + case 0: + // No nanos. + break; + case 1: + optionalNanos = String.format(".%09d", + Integer.valueOf(r.nextInt((int) NANOSECONDS_PER_SECOND))); + break; + case 2: + // Limit to milliseconds only... + optionalNanos = String.format(".%09d", + Integer.valueOf(r.nextInt((int) MILLISECONDS_PER_SECOND)) * NANOSECONDS_PER_MILLISSECOND); + break; + case 3: + // Limit to below milliseconds only... + optionalNanos = String.format(".%09d", + Integer.valueOf(r.nextInt((int) NANOSECONDS_PER_MILLISSECOND))); + break; + } + String timestampStr = String.format("%04d-%02d-%02d %02d:%02d:%02d%s", + Integer.valueOf(minYear + r.nextInt(maxYear - minYear + 1)), // year + Integer.valueOf(1 + r.nextInt(12)), // month + Integer.valueOf(1 + r.nextInt(28)), // day + Integer.valueOf(0 + r.nextInt(24)), // hour + Integer.valueOf(0 + r.nextInt(60)), // minute + Integer.valueOf(0 + r.nextInt(60)), // second + optionalNanos); + Timestamp timestampVal; + try { + timestampVal = Timestamp.valueOf(timestampStr); + } catch (Exception e) { + System.err.println("Timestamp string " + timestampStr + " did not parse"); + throw e; + } + return timestampVal; + } + + public static long randomMillis(long minMillis, long maxMillis, Random rand) { + return minMillis + (long) ((maxMillis - minMillis) * rand.nextDouble()); + } + + public static long randomMillis(Random rand) { + return randomMillis(MIN_FOUR_DIGIT_YEAR_MILLIS, MAX_FOUR_DIGIT_YEAR_MILLIS, rand); + } + + public static int randomNanos(Random rand, int decimalDigits) { + // Only keep the most significant decimalDigits digits. + int nanos = rand.nextInt((int) NANOSECONDS_PER_SECOND); + return nanos - nanos % (int) Math.pow(10, 9 - decimalDigits); + } + + public static int randomNanos(Random rand) { + return randomNanos(rand, 9); + } +} \ No newline at end of file diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..c4f19cf1df49bba789432aa40931ae27d6619f0e --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java @@ -0,0 +1,486 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + + +/** + * This class supports string and binary data by value reference -- i.e. each field is + * explicitly present, as opposed to provided by a dictionary reference. + * In some cases, all the values will be in the same byte array to begin with, + * but this need not be the case. If each value is in a separate byte + * array to start with, or not all of the values are in the same original + * byte array, you can still assign data by reference into this column vector. + * This gives flexibility to use this in multiple situations. + *

+ * When setting data by reference, the caller + * is responsible for allocating the byte arrays used to hold the data. + * You can also set data by value, as long as you call the initBuffer() method first. + * You can mix "by value" and "by reference" in the same column vector, + * though that use is probably not typical. + */ +public class BytesColumnVector extends ColumnVector { + public byte[][] vector; + public int[] start; // start offset of each field + + /* + * The length of each field. If the value repeats for every entry, then it is stored + * in vector[0] and isRepeating from the superclass is set to true. + */ + public int[] length; + + // A call to increaseBufferSpace() or ensureValPreallocated() will ensure that buffer[] points to + // a byte[] with sufficient space for the specified size. + private byte[] buffer; // optional buffer to use when actually copying in data + private int nextFree; // next free position in buffer + + // Hang onto a byte array for holding smaller byte values + private byte[] smallBuffer; + private int smallBufferNextFree; + + private int bufferAllocationCount; + + // Estimate that there will be 16 bytes per entry + static final int DEFAULT_BUFFER_SIZE = 16 * VectorizedRowBatch.DEFAULT_SIZE; + + // Proportion of extra space to provide when allocating more buffer space. + static final float EXTRA_SPACE_FACTOR = (float) 1.2; + + // Largest size allowed in smallBuffer + static final int MAX_SIZE_FOR_SMALL_BUFFER = 1024 * 1024; + + /** + * Use this constructor for normal operation. + * All column vectors should be the default size normally. + */ + public BytesColumnVector() { + this(VectorizedRowBatch.DEFAULT_SIZE); + } + + /** + * Don't call this constructor except for testing purposes. + * + * @param size number of elements in the column vector + */ + public BytesColumnVector(int size) { + super(Type.BYTES, size); + vector = new byte[size][]; + start = new int[size]; + length = new int[size]; + } + + /** + * Additional reset work for BytesColumnVector (releasing scratch bytes for by value strings). + */ + @Override + public void reset() { + super.reset(); + initBuffer(0); + } + + /** Set a field by reference. + * + * @param elementNum index within column vector to set + * @param sourceBuf container of source data + * @param start start byte position within source + * @param length length of source byte sequence + */ + public void setRef(int elementNum, byte[] sourceBuf, int start, int length) { + vector[elementNum] = sourceBuf; + this.start[elementNum] = start; + this.length[elementNum] = length; + } + + /** + * You must call initBuffer first before using setVal(). + * Provide the estimated number of bytes needed to hold + * a full column vector worth of byte string data. + * + * @param estimatedValueSize Estimated size of buffer space needed + */ + public void initBuffer(int estimatedValueSize) { + nextFree = 0; + smallBufferNextFree = 0; + + // if buffer is already allocated, keep using it, don't re-allocate + if (buffer != null) { + // Free up any previously allocated buffers that are referenced by vector + if (bufferAllocationCount > 0) { + for (int idx = 0; idx < vector.length; ++idx) { + vector[idx] = null; + } + buffer = smallBuffer; // In case last row was a large bytes value + } + } else { + // allocate a little extra space to limit need to re-allocate + int bufferSize = this.vector.length * (int)(estimatedValueSize * EXTRA_SPACE_FACTOR); + if (bufferSize < DEFAULT_BUFFER_SIZE) { + bufferSize = DEFAULT_BUFFER_SIZE; + } + buffer = new byte[bufferSize]; + smallBuffer = buffer; + } + bufferAllocationCount = 0; + } + + /** + * Initialize buffer to default size. + */ + public void initBuffer() { + initBuffer(0); + } + + /** + * @return amount of buffer space currently allocated + */ + public int bufferSize() { + if (buffer == null) { + return 0; + } + return buffer.length; + } + + /** + * Set a field by actually copying in to a local buffer. + * If you must actually copy data in to the array, use this method. + * DO NOT USE this method unless it's not practical to set data by reference with setRef(). + * Setting data by reference tends to run a lot faster than copying data in. + * + * @param elementNum index within column vector to set + * @param sourceBuf container of source data + * @param start start byte position within source + * @param length length of source byte sequence + */ + public void setVal(int elementNum, byte[] sourceBuf, int start, int length) { + if ((nextFree + length) > buffer.length) { + increaseBufferSpace(length); + } + System.arraycopy(sourceBuf, start, buffer, nextFree, length); + vector[elementNum] = buffer; + this.start[elementNum] = nextFree; + this.length[elementNum] = length; + nextFree += length; + } + + /** + * Set a field by actually copying in to a local buffer. + * If you must actually copy data in to the array, use this method. + * DO NOT USE this method unless it's not practical to set data by reference with setRef(). + * Setting data by reference tends to run a lot faster than copying data in. + * + * @param elementNum index within column vector to set + * @param sourceBuf container of source data + */ + public void setVal(int elementNum, byte[] sourceBuf) { + setVal(elementNum, sourceBuf, 0, sourceBuf.length); + } + + /** + * Preallocate space in the local buffer so the caller can fill in the value bytes themselves. + * + * Always use with getValPreallocatedBytes, getValPreallocatedStart, and setValPreallocated. + */ + public void ensureValPreallocated(int length) { + if ((nextFree + length) > buffer.length) { + increaseBufferSpace(length); + } + } + + public byte[] getValPreallocatedBytes() { + return buffer; + } + + public int getValPreallocatedStart() { + return nextFree; + } + + /** + * Set the length of the preallocated values bytes used. + * @param elementNum + * @param length + */ + public void setValPreallocated(int elementNum, int length) { + vector[elementNum] = buffer; + this.start[elementNum] = nextFree; + this.length[elementNum] = length; + nextFree += length; + } + + /** + * Set a field to the concatenation of two string values. Result data is copied + * into the internal buffer. + * + * @param elementNum index within column vector to set + * @param leftSourceBuf container of left argument + * @param leftStart start of left argument + * @param leftLen length of left argument + * @param rightSourceBuf container of right argument + * @param rightStart start of right argument + * @param rightLen length of right arugment + */ + public void setConcat(int elementNum, byte[] leftSourceBuf, int leftStart, int leftLen, + byte[] rightSourceBuf, int rightStart, int rightLen) { + int newLen = leftLen + rightLen; + if ((nextFree + newLen) > buffer.length) { + increaseBufferSpace(newLen); + } + vector[elementNum] = buffer; + this.start[elementNum] = nextFree; + this.length[elementNum] = newLen; + + System.arraycopy(leftSourceBuf, leftStart, buffer, nextFree, leftLen); + nextFree += leftLen; + System.arraycopy(rightSourceBuf, rightStart, buffer, nextFree, rightLen); + nextFree += rightLen; + } + + /** + * Increase buffer space enough to accommodate next element. + * This uses an exponential increase mechanism to rapidly + * increase buffer size to enough to hold all data. + * As batches get re-loaded, buffer space allocated will quickly + * stabilize. + * + * @param nextElemLength size of next element to be added + */ + public void increaseBufferSpace(int nextElemLength) { + // A call to increaseBufferSpace() or ensureValPreallocated() will ensure that buffer[] points to + // a byte[] with sufficient space for the specified size. + // This will either point to smallBuffer, or to a newly allocated byte array for larger values. + + if (nextElemLength > MAX_SIZE_FOR_SMALL_BUFFER) { + // Larger allocations will be special-cased and will not use the normal buffer. + // buffer/nextFree will be set to a newly allocated array just for the current row. + // The next row will require another call to increaseBufferSpace() since this new buffer should be used up. + byte[] newBuffer = new byte[nextElemLength]; + ++bufferAllocationCount; + // If the buffer was pointing to smallBuffer, then nextFree keeps track of the current state + // of the free index for smallBuffer. We now need to save this value to smallBufferNextFree + // so we don't lose this. A bit of a weird dance here. + if (smallBuffer == buffer) { + smallBufferNextFree = nextFree; + } + buffer = newBuffer; + nextFree = 0; + } else { + // This value should go into smallBuffer. + if (smallBuffer != buffer) { + // Previous row was for a large bytes value ( > MAX_SIZE_FOR_SMALL_BUFFER). + // Use smallBuffer if possible. + buffer = smallBuffer; + nextFree = smallBufferNextFree; + } + + // smallBuffer might still be out of space + if ((nextFree + nextElemLength) > buffer.length) { + int newLength = smallBuffer.length * 2; + while (newLength < nextElemLength) { + if (newLength < 0) { + throw new RuntimeException("Overflow of newLength. smallBuffer.length=" + + smallBuffer.length + ", nextElemLength=" + nextElemLength); + } + newLength *= 2; + } + smallBuffer = new byte[newLength]; + ++bufferAllocationCount; + smallBufferNextFree = 0; + // Update buffer + buffer = smallBuffer; + nextFree = 0; + } + } + } + + /** Copy the current object contents into the output. Only copy selected entries, + * as indicated by selectedInUse and the sel array. + */ + public void copySelected( + boolean selectedInUse, int[] sel, int size, BytesColumnVector output) { + + // Output has nulls if and only if input has nulls. + output.noNulls = noNulls; + output.isRepeating = false; + + // Handle repeating case + if (isRepeating) { + output.setVal(0, vector[0], start[0], length[0]); + output.isNull[0] = isNull[0]; + output.isRepeating = true; + return; + } + + // Handle normal case + + // Copy data values over + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.setVal(i, vector[i], start[i], length[i]); + } + } + else { + for (int i = 0; i < size; i++) { + output.setVal(i, vector[i], start[i], length[i]); + } + } + + // Copy nulls over if needed + if (!noNulls) { + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.isNull[i] = isNull[i]; + } + } + else { + System.arraycopy(isNull, 0, output.isNull, 0, size); + } + } + } + + /** Simplify vector by brute-force flattening noNulls and isRepeating + * This can be used to reduce combinatorial explosion of code paths in VectorExpressions + * with many arguments, at the expense of loss of some performance. + */ + public void flatten(boolean selectedInUse, int[] sel, int size) { + flattenPush(); + if (isRepeating) { + isRepeating = false; + + // setRef is used below and this is safe, because the reference + // is to data owned by this column vector. If this column vector + // gets re-used, the whole thing is re-used together so there + // is no danger of a dangling reference. + + // Only copy data values if entry is not null. The string value + // at position 0 is undefined if the position 0 value is null. + if (noNulls || !isNull[0]) { + + // loops start at position 1 because position 0 is already set + if (selectedInUse) { + for (int j = 1; j < size; j++) { + int i = sel[j]; + this.setRef(i, vector[0], start[0], length[0]); + } + } else { + for (int i = 1; i < size; i++) { + this.setRef(i, vector[0], start[0], length[0]); + } + } + } + flattenRepeatingNulls(selectedInUse, sel, size); + } + flattenNoNulls(selectedInUse, sel, size); + } + + // Fill the all the vector entries with provided value + public void fill(byte[] value) { + noNulls = true; + isRepeating = true; + setRef(0, value, 0, value.length); + } + + // Fill the column vector with nulls + public void fillWithNulls() { + noNulls = false; + isRepeating = true; + vector[0] = null; + isNull[0] = true; + } + + @Override + public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) { + if (inputVector.isRepeating) { + inputElementNum = 0; + } + if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) { + isNull[outElementNum] = false; + BytesColumnVector in = (BytesColumnVector) inputVector; + setVal(outElementNum, in.vector[inputElementNum], + in.start[inputElementNum], in.length[inputElementNum]); + } else { + isNull[outElementNum] = true; + noNulls = false; + } + } + + @Override + public void init() { + initBuffer(0); + } + + public String toString(int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + return new String(vector[row], start[row], length[row]); + } else { + return null; + } + } + + @Override + public void stringifyValue(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + buffer.append('"'); + buffer.append(new String(vector[row], start[row], length[row])); + buffer.append('"'); + } else { + buffer.append("null"); + } + } + + @Override + public void ensureSize(int size, boolean preserveData) { + super.ensureSize(size, preserveData); + if (size > vector.length) { + int[] oldStart = start; + start = new int[size]; + int[] oldLength = length; + length = new int[size]; + byte[][] oldVector = vector; + vector = new byte[size][]; + if (preserveData) { + if (isRepeating) { + vector[0] = oldVector[0]; + start[0] = oldStart[0]; + length[0] = oldLength[0]; + } else { + System.arraycopy(oldVector, 0, vector, 0, oldVector.length); + System.arraycopy(oldStart, 0, start, 0 , oldStart.length); + System.arraycopy(oldLength, 0, length, 0, oldLength.length); + } + } + } + } + + @Override + public void shallowCopyTo(ColumnVector otherCv) { + BytesColumnVector other = (BytesColumnVector)otherCv; + super.shallowCopyTo(other); + other.nextFree = nextFree; + other.vector = vector; + other.start = start; + other.length = length; + other.buffer = buffer; + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..57342729fe2b1d2e20d3e51088d59ac3fe4fbd83 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + +import java.util.Arrays; + +/** + * ColumnVector contains the shared structure for the sub-types, + * including NULL information, and whether this vector + * repeats, i.e. has all values the same, so only the first + * one is set. This is used to accelerate query performance + * by handling a whole vector in O(1) time when applicable. + * + * The fields are public by design since this is a performance-critical + * structure that is used in the inner loop of query execution. + */ +public abstract class ColumnVector { + + /* + * The current kinds of column vectors. + */ + public static enum Type { + NONE, // Useful when the type of column vector has not be determined yet. + LONG, + DOUBLE, + BYTES, + DECIMAL, + DECIMAL_64, + TIMESTAMP, + INTERVAL_DAY_TIME, + STRUCT, + LIST, + MAP, + UNION + } + + public final Type type; + + /* + * If hasNulls is true, then this array contains true if the value + * is null, otherwise false. The array is always allocated, so a batch can be re-used + * later and nulls added. + */ + public boolean[] isNull; + + // If the whole column vector has no nulls, this is true, otherwise false. + public boolean noNulls; + + /* + * True if same value repeats for whole column vector. + * If so, vector[0] holds the repeating value. + */ + public boolean isRepeating; + + // Variables to hold state from before flattening so it can be easily restored. + private boolean preFlattenIsRepeating; + private boolean preFlattenNoNulls; + + /** + * Constructor for super-class ColumnVector. This is not called directly, + * but used to initialize inherited fields. + * + * @param len Vector length + */ + public ColumnVector(Type type, int len) { + this.type = type; + isNull = new boolean[len]; + noNulls = true; + isRepeating = false; + preFlattenNoNulls = true; + preFlattenIsRepeating = false; + } + + /** + * Resets the column to default state + * - fills the isNull array with false + * - sets noNulls to true + * - sets isRepeating to false + */ + public void reset() { + if (!noNulls) { + Arrays.fill(isNull, false); + } + noNulls = true; + isRepeating = false; + preFlattenNoNulls = true; + preFlattenIsRepeating = false; + } + + /** + * Sets the isRepeating flag. Recurses over structs and unions so that the + * flags are set correctly. + * @param isRepeating + */ + public void setRepeating(boolean isRepeating) { + this.isRepeating = isRepeating; + } + + abstract public void flatten(boolean selectedInUse, int[] sel, int size); + + // Simplify vector by brute-force flattening noNulls if isRepeating + // This can be used to reduce combinatorial explosion of code paths in VectorExpressions + // with many arguments. + protected void flattenRepeatingNulls(boolean selectedInUse, int[] sel, + int size) { + + boolean nullFillValue; + + if (noNulls) { + nullFillValue = false; + } else { + nullFillValue = isNull[0]; + } + + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + isNull[i] = nullFillValue; + } + } else { + Arrays.fill(isNull, 0, size, nullFillValue); + } + + // all nulls are now explicit + noNulls = false; + } + + protected void flattenNoNulls(boolean selectedInUse, int[] sel, + int size) { + if (noNulls) { + noNulls = false; + if (selectedInUse) { + for (int j = 0; j < size; j++) { + isNull[sel[j]] = false; + } + } else { + Arrays.fill(isNull, 0, size, false); + } + } + } + + /** + * Restore the state of isRepeating and noNulls to what it was + * before flattening. This must only be called just after flattening + * and then evaluating a VectorExpression on the column vector. + * It is an optimization that allows other operations on the same + * column to continue to benefit from the isRepeating and noNulls + * indicators. + */ + public void unFlatten() { + isRepeating = preFlattenIsRepeating; + noNulls = preFlattenNoNulls; + } + + // Record repeating and no nulls state to be restored later. + protected void flattenPush() { + preFlattenIsRepeating = isRepeating; + preFlattenNoNulls = noNulls; + } + + /** + * Set the element in this column vector from the given input vector. + * This method can assume that the output does not have isRepeating set. + */ + public abstract void setElement(int outElementNum, int inputElementNum, + ColumnVector inputVector); + + /** + * Initialize the column vector. This method can be overridden by specific column vector types. + * Use this method only if the individual type of the column vector is not known, otherwise its + * preferable to call specific initialization methods. + */ + public void init() { + // Do nothing by default + } + + /** + * Ensure the ColumnVector can hold at least size values. + * This method is deliberately *not* recursive because the complex types + * can easily have more (or less) children than the upper levels. + * @param size the new minimum size + * @param preserveData should the old data be preserved? + */ + public void ensureSize(int size, boolean preserveData) { + if (isNull.length < size) { + boolean[] oldArray = isNull; + isNull = new boolean[size]; + if (preserveData && !noNulls) { + if (isRepeating) { + isNull[0] = oldArray[0]; + } else { + System.arraycopy(oldArray, 0, isNull, 0, oldArray.length); + } + } + } + } + + /** + * Print the value for this column into the given string builder. + * @param buffer the buffer to print into + * @param row the id of the row to print + */ + public abstract void stringifyValue(StringBuilder buffer, + int row); + + /** + * Shallow copy of the contents of this vector to the other vector; + * replaces other vector's values. + */ + public void shallowCopyTo(ColumnVector otherCv) { + otherCv.isNull = isNull; + otherCv.noNulls = noNulls; + otherCv.isRepeating = isRepeating; + otherCv.preFlattenIsRepeating = preFlattenIsRepeating; + otherCv.preFlattenNoNulls = preFlattenNoNulls; + } + } diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..5548b9dfbfd08efac0119427165f2981fb4f2229 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec.vector; + +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; + +/** + + */ +public class Decimal64ColumnVector extends LongColumnVector { + + public short scale; + public short precision; + + private HiveDecimalWritable tempHiveDecWritable; + + public Decimal64ColumnVector(int precision, int scale) { + this(VectorizedRowBatch.DEFAULT_SIZE, precision, scale); + } + + public Decimal64ColumnVector(int size, int precision, int scale) { + super(size); + this.precision = (short) precision; + this.scale = (short) scale; + tempHiveDecWritable = new HiveDecimalWritable(); + } + + public void set(int elementNum, HiveDecimalWritable writable) { + tempHiveDecWritable.set(writable); + tempHiveDecWritable.mutateEnforcePrecisionScale(precision, scale); + if (!tempHiveDecWritable.isSet()) { + noNulls = false; + isNull[elementNum] = true; + } else { + isNull[elementNum] = false; + vector[elementNum] = tempHiveDecWritable.serialize64(scale); + } + } + + public void set(int elementNum, HiveDecimal hiveDec) { + tempHiveDecWritable.set(hiveDec); + tempHiveDecWritable.mutateEnforcePrecisionScale(precision, scale); + if (!tempHiveDecWritable.isSet()) { + noNulls = false; + isNull[elementNum] = true; + } else { + isNull[elementNum] = false; + vector[elementNum] = tempHiveDecWritable.serialize64(scale); + } + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..cfe40aca4908701b8d71955f0d52b3cb1f1a72cd --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + + +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.common.type.HiveDecimal; + +public class DecimalColumnVector extends ColumnVector { + + /** + * A vector of HiveDecimalWritable objects. + * + * For high performance and easy access to this low-level structure, + * the fields are public by design (as they are in other ColumnVector + * types). + */ + public HiveDecimalWritable[] vector; + public short scale; + public short precision; + + public DecimalColumnVector(int precision, int scale) { + this(VectorizedRowBatch.DEFAULT_SIZE, precision, scale); + } + + public DecimalColumnVector(int size, int precision, int scale) { + super(Type.DECIMAL, size); + this.precision = (short) precision; + this.scale = (short) scale; + vector = new HiveDecimalWritable[size]; + for (int i = 0; i < size; i++) { + vector[i] = new HiveDecimalWritable(0); // Initially zero. + } + } + + // Fill the all the vector entries with provided value + public void fill(HiveDecimal value) { + noNulls = true; + isRepeating = true; + if (vector[0] == null) { + vector[0] = new HiveDecimalWritable(value); + } else { + vector[0].set(value); + } + } + + @Override + public void flatten(boolean selectedInUse, int[] sel, int size) { + // TODO Auto-generated method stub + } + + @Override + public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) { + if (inputVector.isRepeating) { + inputElementNum = 0; + } + if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) { + vector[outElementNum].set( + ((DecimalColumnVector) inputVector).vector[inputElementNum], + precision, scale); + if (!vector[outElementNum].isSet()) { + isNull[outElementNum] = true; + noNulls = false; + } else { + isNull[outElementNum] = false; + } + } else { + isNull[outElementNum] = true; + noNulls = false; + } + } + + @Override + public void stringifyValue(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + buffer.append(vector[row].toString()); + } else { + buffer.append("null"); + } + } + + public void set(int elementNum, HiveDecimalWritable writeable) { + vector[elementNum].set(writeable, precision, scale); + if (!vector[elementNum].isSet()) { + noNulls = false; + isNull[elementNum] = true; + } else { + isNull[elementNum] = false; + } + } + + public void set(int elementNum, HiveDecimal hiveDec) { + vector[elementNum].set(hiveDec, precision, scale); + if (!vector[elementNum].isSet()) { + noNulls = false; + isNull[elementNum] = true; + } else { + isNull[elementNum] = false; + } + } + + public void setNullDataValue(int elementNum) { + // E.g. For scale 2 the minimum is "0.01" + vector[elementNum].setFromLongAndScale(1L, scale); + } + + @Override + public void ensureSize(int size, boolean preserveData) { + super.ensureSize(size, preserveData); + if (size <= vector.length) return; // We assume the existing vector is always valid. + HiveDecimalWritable[] oldArray = vector; + vector = new HiveDecimalWritable[size]; + int initPos = 0; + if (preserveData) { + // we copy all of the values to avoid creating more objects + // TODO: it might be cheaper to always preserve data or reset existing objects + initPos = oldArray.length; + System.arraycopy(oldArray, 0, vector, 0 , oldArray.length); + } + for (int i = initPos; i < vector.length; ++i) { + vector[i] = new HiveDecimalWritable(0); // Initially zero. + } + } + + @Override + public void shallowCopyTo(ColumnVector otherCv) { + DecimalColumnVector other = (DecimalColumnVector)otherCv; + super.shallowCopyTo(other); + other.scale = scale; + other.precision = precision; + other.vector = vector; + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..1395144ddde598508995d0a1077b8c19d2d6c510 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java @@ -0,0 +1,184 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec.vector; + +import java.util.Arrays; + +/** + * This class represents a nullable double precision floating point column vector. + * This class will be used for operations on all floating point types (float, double) + * and as such will use a 64-bit double value to hold the biggest possible value. + * During copy-in/copy-out, smaller types (i.e. float) will be converted as needed. This will + * reduce the amount of code that needs to be generated and also will run fast since the + * machine operates with 64-bit words. + * + * The vector[] field is public by design for high-performance access in the inner + * loop of query execution. + */ +public class DoubleColumnVector extends ColumnVector { + public double[] vector; + public static final double NULL_VALUE = Double.NaN; + + /** + * Use this constructor by default. All column vectors + * should normally be the default size. + */ + public DoubleColumnVector() { + this(VectorizedRowBatch.DEFAULT_SIZE); + } + + /** + * Don't use this except for testing purposes. + * + * @param len + */ + public DoubleColumnVector(int len) { + super(Type.DOUBLE, len); + vector = new double[len]; + } + + // Copy the current object contents into the output. Only copy selected entries, + // as indicated by selectedInUse and the sel array. + public void copySelected( + boolean selectedInUse, int[] sel, int size, DoubleColumnVector output) { + + // Output has nulls if and only if input has nulls. + output.noNulls = noNulls; + output.isRepeating = false; + + // Handle repeating case + if (isRepeating) { + output.vector[0] = vector[0]; + output.isNull[0] = isNull[0]; + output.isRepeating = true; + return; + } + + // Handle normal case + + // Copy data values over + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.vector[i] = vector[i]; + } + } + else { + System.arraycopy(vector, 0, output.vector, 0, size); + } + + // Copy nulls over if needed + if (!noNulls) { + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.isNull[i] = isNull[i]; + } + } + else { + System.arraycopy(isNull, 0, output.isNull, 0, size); + } + } + } + + // Fill the column vector with the provided value + public void fill(double value) { + noNulls = true; + isRepeating = true; + vector[0] = value; + } + + // Fill the column vector with nulls + public void fillWithNulls() { + noNulls = false; + isRepeating = true; + vector[0] = NULL_VALUE; + isNull[0] = true; + } + + // Simplify vector by brute-force flattening noNulls and isRepeating + // This can be used to reduce combinatorial explosion of code paths in VectorExpressions + // with many arguments. + public void flatten(boolean selectedInUse, int[] sel, int size) { + flattenPush(); + if (isRepeating) { + isRepeating = false; + double repeatVal = vector[0]; + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + vector[i] = repeatVal; + } + } else { + Arrays.fill(vector, 0, size, repeatVal); + } + flattenRepeatingNulls(selectedInUse, sel, size); + } + flattenNoNulls(selectedInUse, sel, size); + } + + @Override + public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) { + if (inputVector.isRepeating) { + inputElementNum = 0; + } + if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) { + isNull[outElementNum] = false; + vector[outElementNum] = + ((DoubleColumnVector) inputVector).vector[inputElementNum]; + } else { + isNull[outElementNum] = true; + noNulls = false; + } + } + + @Override + public void stringifyValue(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + buffer.append(vector[row]); + } else { + buffer.append("null"); + } + } + + @Override + public void ensureSize(int size, boolean preserveData) { + super.ensureSize(size, preserveData); + if (size > vector.length) { + double[] oldArray = vector; + vector = new double[size]; + if (preserveData) { + if (isRepeating) { + vector[0] = oldArray[0]; + } else { + System.arraycopy(oldArray, 0, vector, 0 , oldArray.length); + } + } + } + } + + @Override + public void shallowCopyTo(ColumnVector otherCv) { + DoubleColumnVector other = (DoubleColumnVector)otherCv; + super.shallowCopyTo(other); + other.vector = vector; + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..587e2b9ac8454174109c3b437e539876bc5b6ce0 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java @@ -0,0 +1,375 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec.vector; + +import java.util.Arrays; + +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; +import org.apache.hadoop.io.Writable; + +/** + * This class represents a nullable interval day time column vector capable of handing a + * wide range of interval day time values. + * + * We store the 2 (value) fields of a HiveIntervalDayTime class in primitive arrays. + * + * We do this to avoid an array of Java HiveIntervalDayTime objects which would have poor storage + * and memory access characteristics. + * + * Generally, the caller will fill in a scratch HiveIntervalDayTime object with values from a row, + * work using the scratch HiveIntervalDayTime, and then perhaps update the column vector row + * with a result. + */ +public class IntervalDayTimeColumnVector extends ColumnVector { + + /* + * The storage arrays for this column vector corresponds to the storage of a HiveIntervalDayTime: + */ + private long[] totalSeconds; + // The values from HiveIntervalDayTime.getTotalSeconds(). + + private int[] nanos; + // The values from HiveIntervalDayTime.getNanos(). + + /* + * Scratch objects. + */ + private final HiveIntervalDayTime scratchIntervalDayTime; + + private Writable scratchWritable; + // Supports keeping a HiveIntervalDayTimeWritable object without having to import + // that definition... + + /** + * Use this constructor by default. All column vectors + * should normally be the default size. + */ + public IntervalDayTimeColumnVector() { + this(VectorizedRowBatch.DEFAULT_SIZE); + } + + /** + * Don't use this except for testing purposes. + * + * @param len the number of rows + */ + public IntervalDayTimeColumnVector(int len) { + super(Type.INTERVAL_DAY_TIME, len); + + totalSeconds = new long[len]; + nanos = new int[len]; + + scratchIntervalDayTime = new HiveIntervalDayTime(); + + scratchWritable = null; // Allocated by caller. + } + + /** + * Return the number of rows. + * @return + */ + public int getLength() { + return totalSeconds.length; + } + + /** + * Return a row's HiveIntervalDayTime.getTotalSeconds() value. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @return + */ + public long getTotalSeconds(int elementNum) { + return totalSeconds[elementNum]; + } + + /** + * Return a row's HiveIntervalDayTime.getNanos() value. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @return + */ + public long getNanos(int elementNum) { + return nanos[elementNum]; + } + + /** + * Return a row's HiveIntervalDayTime.getDouble() value. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @return + */ + public double getDouble(int elementNum) { + return asScratchIntervalDayTime(elementNum).getDouble(); + } + + /** + * Set a HiveIntervalDayTime object from a row of the column. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param intervalDayTime + * @param elementNum + */ + public void intervalDayTimeUpdate(HiveIntervalDayTime intervalDayTime, int elementNum) { + intervalDayTime.set(totalSeconds[elementNum], nanos[elementNum]); + } + + + /** + * Return the scratch HiveIntervalDayTime object set from a row. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @return + */ + public HiveIntervalDayTime asScratchIntervalDayTime(int elementNum) { + scratchIntervalDayTime.set(totalSeconds[elementNum], nanos[elementNum]); + return scratchIntervalDayTime; + } + + /** + * Return the scratch HiveIntervalDayTime (contents undefined). + * @return + */ + public HiveIntervalDayTime getScratchIntervalDayTime() { + return scratchIntervalDayTime; + } + + /** + * Compare row to HiveIntervalDayTime. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @param intervalDayTime + * @return -1, 0, 1 standard compareTo values. + */ + public int compareTo(int elementNum, HiveIntervalDayTime intervalDayTime) { + return asScratchIntervalDayTime(elementNum).compareTo(intervalDayTime); + } + + /** + * Compare HiveIntervalDayTime to row. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param intervalDayTime + * @param elementNum + * @return -1, 0, 1 standard compareTo values. + */ + public int compareTo(HiveIntervalDayTime intervalDayTime, int elementNum) { + return intervalDayTime.compareTo(asScratchIntervalDayTime(elementNum)); + } + + /** + * Compare a row to another TimestampColumnVector's row. + * @param elementNum1 + * @param intervalDayTimeColVector2 + * @param elementNum2 + * @return + */ + public int compareTo(int elementNum1, IntervalDayTimeColumnVector intervalDayTimeColVector2, + int elementNum2) { + return asScratchIntervalDayTime(elementNum1).compareTo( + intervalDayTimeColVector2.asScratchIntervalDayTime(elementNum2)); + } + + /** + * Compare another TimestampColumnVector's row to a row. + * @param intervalDayTimeColVector1 + * @param elementNum1 + * @param elementNum2 + * @return + */ + public int compareTo(IntervalDayTimeColumnVector intervalDayTimeColVector1, int elementNum1, + int elementNum2) { + return intervalDayTimeColVector1.asScratchIntervalDayTime(elementNum1).compareTo( + asScratchIntervalDayTime(elementNum2)); + } + + @Override + public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) { + + IntervalDayTimeColumnVector timestampColVector = (IntervalDayTimeColumnVector) inputVector; + + totalSeconds[outElementNum] = timestampColVector.totalSeconds[inputElementNum]; + nanos[outElementNum] = timestampColVector.nanos[inputElementNum]; + } + + // Simplify vector by brute-force flattening noNulls and isRepeating + // This can be used to reduce combinatorial explosion of code paths in VectorExpressions + // with many arguments. + public void flatten(boolean selectedInUse, int[] sel, int size) { + flattenPush(); + if (isRepeating) { + isRepeating = false; + long repeatFastTime = totalSeconds[0]; + int repeatNanos = nanos[0]; + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + totalSeconds[i] = repeatFastTime; + nanos[i] = repeatNanos; + } + } else { + Arrays.fill(totalSeconds, 0, size, repeatFastTime); + Arrays.fill(nanos, 0, size, repeatNanos); + } + flattenRepeatingNulls(selectedInUse, sel, size); + } + flattenNoNulls(selectedInUse, sel, size); + } + + /** + * Set a row from a HiveIntervalDayTime. + * We assume the entry has already been isRepeated adjusted. + * @param elementNum + * @param intervalDayTime + */ + public void set(int elementNum, HiveIntervalDayTime intervalDayTime) { + this.totalSeconds[elementNum] = intervalDayTime.getTotalSeconds(); + this.nanos[elementNum] = intervalDayTime.getNanos(); + } + + /** + * Set a row from the current value in the scratch interval day time. + * @param elementNum + */ + public void setFromScratchIntervalDayTime(int elementNum) { + this.totalSeconds[elementNum] = scratchIntervalDayTime.getTotalSeconds(); + this.nanos[elementNum] = scratchIntervalDayTime.getNanos(); + } + + /** + * Set row to standard null value(s). + * We assume the entry has already been isRepeated adjusted. + * @param elementNum + */ + public void setNullValue(int elementNum) { + totalSeconds[elementNum] = 0; + nanos[elementNum] = 1; + } + + // Copy the current object contents into the output. Only copy selected entries, + // as indicated by selectedInUse and the sel array. + public void copySelected( + boolean selectedInUse, int[] sel, int size, IntervalDayTimeColumnVector output) { + + // Output has nulls if and only if input has nulls. + output.noNulls = noNulls; + output.isRepeating = false; + + // Handle repeating case + if (isRepeating) { + output.totalSeconds[0] = totalSeconds[0]; + output.nanos[0] = nanos[0]; + output.isNull[0] = isNull[0]; + output.isRepeating = true; + return; + } + + // Handle normal case + + // Copy data values over + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.totalSeconds[i] = totalSeconds[i]; + output.nanos[i] = nanos[i]; + } + } + else { + System.arraycopy(totalSeconds, 0, output.totalSeconds, 0, size); + System.arraycopy(nanos, 0, output.nanos, 0, size); + } + + // Copy nulls over if needed + if (!noNulls) { + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.isNull[i] = isNull[i]; + } + } + else { + System.arraycopy(isNull, 0, output.isNull, 0, size); + } + } + } + + /** + * Fill all the vector entries with a HiveIntervalDayTime. + * @param intervalDayTime + */ + public void fill(HiveIntervalDayTime intervalDayTime) { + noNulls = true; + isRepeating = true; + totalSeconds[0] = intervalDayTime.getTotalSeconds(); + nanos[0] = intervalDayTime.getNanos(); + } + + /** + * Return a convenience writable object stored by this column vector. + * Supports keeping a TimestampWritable object without having to import that definition... + * @return + */ + public Writable getScratchWritable() { + return scratchWritable; + } + + /** + * Set the convenience writable object stored by this column vector + * @param scratchWritable + */ + public void setScratchWritable(Writable scratchWritable) { + this.scratchWritable = scratchWritable; + } + + @Override + public void stringifyValue(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + scratchIntervalDayTime.set(totalSeconds[row], nanos[row]); + buffer.append(scratchIntervalDayTime.toString()); + } else { + buffer.append("null"); + } + } + + @Override + public void ensureSize(int size, boolean preserveData) { + super.ensureSize(size, preserveData); + if (size <= totalSeconds.length) return; + long[] oldTime = totalSeconds; + int[] oldNanos = nanos; + totalSeconds = new long[size]; + nanos = new int[size]; + if (preserveData) { + if (isRepeating) { + totalSeconds[0] = oldTime[0]; + nanos[0] = oldNanos[0]; + } else { + System.arraycopy(oldTime, 0, totalSeconds, 0, oldTime.length); + System.arraycopy(oldNanos, 0, nanos, 0, oldNanos.length); + } + } + } + + @Override + public void shallowCopyTo(ColumnVector otherCv) { + IntervalDayTimeColumnVector other = (IntervalDayTimeColumnVector)otherCv; + super.shallowCopyTo(other); + other.totalSeconds = totalSeconds; + other.nanos = nanos; + } +} \ No newline at end of file diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ListColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ListColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..02a8b3cec5f5cad98e4802e6e1f1e9ece268921f --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ListColumnVector.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + +/** + * The representation of a vectorized column of list objects. + * + * Each list is composed of a range of elements in the underlying child + * ColumnVector. The range for list i is + * offsets[i]..offsets[i]+lengths[i]-1 inclusive. + */ +public class ListColumnVector extends MultiValuedColumnVector { + + public ColumnVector child; + + public ListColumnVector() { + this(VectorizedRowBatch.DEFAULT_SIZE, null); + } + + /** + * Constructor for ListColumnVector. + * + * @param len Vector length + * @param child The child vector + */ + public ListColumnVector(int len, ColumnVector child) { + super(Type.LIST, len); + this.child = child; + } + + @Override + protected void childFlatten(boolean useSelected, int[] selected, int size) { + child.flatten(useSelected, selected, size); + } + + @Override + public void setElement(int outElementNum, int inputElementNum, + ColumnVector inputVector) { + ListColumnVector input = (ListColumnVector) inputVector; + if (input.isRepeating) { + inputElementNum = 0; + } + if (!input.noNulls && input.isNull[inputElementNum]) { + isNull[outElementNum] = true; + noNulls = false; + } else { + isNull[outElementNum] = false; + int offset = childCount; + int length = (int) input.lengths[inputElementNum]; + int inputOffset = (int) input.offsets[inputElementNum]; + offsets[outElementNum] = offset; + childCount += length; + lengths[outElementNum] = length; + child.ensureSize(childCount, true); + for (int i = 0; i < length; ++i) { + child.setElement(i + offset, inputOffset + i, input.child); + } + } + } + + @Override + public void stringifyValue(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + buffer.append('['); + boolean isFirst = true; + for(long i=offsets[row]; i < offsets[row] + lengths[row]; ++i) { + if (isFirst) { + isFirst = false; + } else { + buffer.append(", "); + } + child.stringifyValue(buffer, (int) i); + } + buffer.append(']'); + } else { + buffer.append("null"); + } + } + + @Override + public void init() { + super.init(); + child.init(); + } + + @Override + public void reset() { + super.reset(); + child.reset(); + } + + @Override + public void unFlatten() { + super.unFlatten(); + if (!isRepeating || noNulls || !isNull[0]) { + child.unFlatten(); + } + } + +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..a407bcdaaaa84e957725783cf84fec06ddcf7143 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java @@ -0,0 +1,230 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec.vector; + +import java.util.Arrays; + +/** + * This class represents a nullable int column vector. + * This class will be used for operations on all integer types (tinyint, smallint, int, bigint) + * and as such will use a 64-bit long value to hold the biggest possible value. + * During copy-in/copy-out, smaller int types will be converted as needed. This will + * reduce the amount of code that needs to be generated and also will run fast since the + * machine operates with 64-bit words. + * + * The vector[] field is public by design for high-performance access in the inner + * loop of query execution. + */ +public class LongColumnVector extends ColumnVector { + public long[] vector; + public static final long NULL_VALUE = 1; + + /** + * Use this constructor by default. All column vectors + * should normally be the default size. + */ + public LongColumnVector() { + this(VectorizedRowBatch.DEFAULT_SIZE); + } + + /** + * Don't use this except for testing purposes. + * + * @param len the number of rows + */ + public LongColumnVector(int len) { + super(Type.LONG, len); + vector = new long[len]; + } + + // Copy the current object contents into the output. Only copy selected entries, + // as indicated by selectedInUse and the sel array. + public void copySelected( + boolean selectedInUse, int[] sel, int size, LongColumnVector output) { + + // Output has nulls if and only if input has nulls. + output.noNulls = noNulls; + output.isRepeating = false; + + // Handle repeating case + if (isRepeating) { + output.vector[0] = vector[0]; + output.isNull[0] = isNull[0]; + output.isRepeating = true; + return; + } + + // Handle normal case + + // Copy data values over + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.vector[i] = vector[i]; + } + } + else { + System.arraycopy(vector, 0, output.vector, 0, size); + } + + // Copy nulls over if needed + if (!noNulls) { + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.isNull[i] = isNull[i]; + } + } + else { + System.arraycopy(isNull, 0, output.isNull, 0, size); + } + } + } + + // Copy the current object contents into the output. Only copy selected entries, + // as indicated by selectedInUse and the sel array. + public void copySelected( + boolean selectedInUse, int[] sel, int size, DoubleColumnVector output) { + + // Output has nulls if and only if input has nulls. + output.noNulls = noNulls; + output.isRepeating = false; + + // Handle repeating case + if (isRepeating) { + output.vector[0] = vector[0]; // automatic conversion to double is done here + output.isNull[0] = isNull[0]; + output.isRepeating = true; + return; + } + + // Handle normal case + + // Copy data values over + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.vector[i] = vector[i]; + } + } + else { + for(int i = 0; i < size; ++i) { + output.vector[i] = vector[i]; + } + } + + // Copy nulls over if needed + if (!noNulls) { + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.isNull[i] = isNull[i]; + } + } + else { + System.arraycopy(isNull, 0, output.isNull, 0, size); + } + } + } + + // Fill the column vector with the provided value + public void fill(long value) { + noNulls = true; + isRepeating = true; + vector[0] = value; + } + + // Fill the column vector with nulls + public void fillWithNulls() { + noNulls = false; + isRepeating = true; + vector[0] = NULL_VALUE; + isNull[0] = true; + } + + // Simplify vector by brute-force flattening noNulls and isRepeating + // This can be used to reduce combinatorial explosion of code paths in VectorExpressions + // with many arguments. + public void flatten(boolean selectedInUse, int[] sel, int size) { + flattenPush(); + if (isRepeating) { + isRepeating = false; + long repeatVal = vector[0]; + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + vector[i] = repeatVal; + } + } else { + Arrays.fill(vector, 0, size, repeatVal); + } + flattenRepeatingNulls(selectedInUse, sel, size); + } + flattenNoNulls(selectedInUse, sel, size); + } + + @Override + public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) { + if (inputVector.isRepeating) { + inputElementNum = 0; + } + if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) { + isNull[outElementNum] = false; + vector[outElementNum] = + ((LongColumnVector) inputVector).vector[inputElementNum]; + } else { + isNull[outElementNum] = true; + noNulls = false; + } + } + + @Override + public void stringifyValue(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + buffer.append(vector[row]); + } else { + buffer.append("null"); + } + } + + @Override + public void ensureSize(int size, boolean preserveData) { + super.ensureSize(size, preserveData); + if (size > vector.length) { + long[] oldArray = vector; + vector = new long[size]; + if (preserveData) { + if (isRepeating) { + vector[0] = oldArray[0]; + } else { + System.arraycopy(oldArray, 0, vector, 0 , oldArray.length); + } + } + } + } + + @Override + public void shallowCopyTo(ColumnVector otherCv) { + LongColumnVector other = (LongColumnVector)otherCv; + super.shallowCopyTo(other); + other.vector = vector; + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MapColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MapColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..64badb99e5385af359691bc1d4df5119c9ffaac3 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MapColumnVector.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + +/** + * The representation of a vectorized column of map objects. + * + * Each map is composed of a range of elements in the underlying child + * ColumnVector. The range for map i is + * offsets[i]..offsets[i]+lengths[i]-1 inclusive. + */ +public class MapColumnVector extends MultiValuedColumnVector { + + public ColumnVector keys; + public ColumnVector values; + + public MapColumnVector() { + this(VectorizedRowBatch.DEFAULT_SIZE, null, null); + } + + /** + * Constructor for MapColumnVector + * + * @param len Vector length + * @param keys The keys column vector + * @param values The values column vector + */ + public MapColumnVector(int len, ColumnVector keys, ColumnVector values) { + super(Type.MAP, len); + this.keys = keys; + this.values = values; + } + + @Override + protected void childFlatten(boolean useSelected, int[] selected, int size) { + keys.flatten(useSelected, selected, size); + values.flatten(useSelected, selected, size); + } + + @Override + public void setElement(int outElementNum, int inputElementNum, + ColumnVector inputVector) { + if (inputVector.isRepeating) { + inputElementNum = 0; + } + if (!inputVector.noNulls && inputVector.isNull[inputElementNum]) { + isNull[outElementNum] = true; + noNulls = false; + } else { + MapColumnVector input = (MapColumnVector) inputVector; + isNull[outElementNum] = false; + int offset = childCount; + int length = (int) input.lengths[inputElementNum]; + int inputOffset = (int) input.offsets[inputElementNum]; + offsets[outElementNum] = offset; + childCount += length; + lengths[outElementNum] = length; + keys.ensureSize(childCount, true); + values.ensureSize(childCount, true); + for (int i = 0; i < length; ++i) { + keys.setElement(i + offset, inputOffset + i, input.keys); + values.setElement(i + offset, inputOffset + i, input.values); + } + } + } + + @Override + public void stringifyValue(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + buffer.append('['); + boolean isFirst = true; + for(long i=offsets[row]; i < offsets[row] + lengths[row]; ++i) { + if (isFirst) { + isFirst = false; + } else { + buffer.append(", "); + } + buffer.append("{\"key\": "); + keys.stringifyValue(buffer, (int) i); + buffer.append(", \"value\": "); + values.stringifyValue(buffer, (int) i); + buffer.append('}'); + } + buffer.append(']'); + } else { + buffer.append("null"); + } + } + + @Override + public void init() { + super.init(); + keys.init(); + values.init(); + } + + @Override + public void reset() { + super.reset(); + keys.reset(); + values.reset(); + } + + @Override + public void unFlatten() { + super.unFlatten(); + if (!isRepeating || noNulls || !isNull[0]) { + keys.unFlatten(); + values.unFlatten(); + } + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MultiValuedColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MultiValuedColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..66136351d4f508d1dc7cf419da8eca41efa698a1 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MultiValuedColumnVector.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + +import java.util.Arrays; + +/** + * The representation of a vectorized column of multi-valued objects, such + * as lists and maps. + * + * Each object is composed of a range of elements in the underlying child + * ColumnVector. The range for list i is + * offsets[i]..offsets[i]+lengths[i]-1 inclusive. + */ +public abstract class MultiValuedColumnVector extends ColumnVector { + + public long[] offsets; + public long[] lengths; + // the number of children slots used + public int childCount; + + /** + * Constructor for MultiValuedColumnVector. + * + * @param len Vector length + */ + public MultiValuedColumnVector(Type type, int len) { + super(type, len); + childCount = 0; + offsets = new long[len]; + lengths = new long[len]; + } + + protected abstract void childFlatten(boolean useSelected, int[] selected, + int size); + + @Override + public void flatten(boolean selectedInUse, int[] sel, int size) { + flattenPush(); + + if (isRepeating) { + if (noNulls || !isNull[0]) { + if (selectedInUse) { + for (int i = 0; i < size; ++i) { + int row = sel[i]; + offsets[row] = offsets[0]; + lengths[row] = lengths[0]; + isNull[row] = false; + } + } else { + Arrays.fill(offsets, 0, size, offsets[0]); + Arrays.fill(lengths, 0, size, lengths[0]); + Arrays.fill(isNull, 0, size, false); + } + // We optimize by assuming that a repeating list/map will run from + // from 0 .. lengths[0] in the child vector. + // Sanity check the assumption that we can start at 0. + if (offsets[0] != 0) { + throw new IllegalArgumentException("Repeating offset isn't 0, but " + + offsets[0]); + } + childFlatten(false, null, (int) lengths[0]); + } else { + if (selectedInUse) { + for(int i=0; i < size; ++i) { + isNull[sel[i]] = true; + } + } else { + Arrays.fill(isNull, 0, size, true); + } + } + isRepeating = false; + noNulls = false; + } else { + if (selectedInUse) { + int childSize = 0; + for(int i=0; i < size; ++i) { + childSize += lengths[sel[i]]; + } + int[] childSelection = new int[childSize]; + int idx = 0; + for(int i=0; i < size; ++i) { + int row = sel[i]; + for(int elem=0; elem < lengths[row]; ++elem) { + childSelection[idx++] = (int) (offsets[row] + elem); + } + } + childFlatten(true, childSelection, childSize); + } else { + childFlatten(false, null, childCount); + } + flattenNoNulls(selectedInUse, sel, size); + } + } + + @Override + public void ensureSize(int size, boolean preserveData) { + super.ensureSize(size, preserveData); + if (size > offsets.length) { + long[] oldOffsets = offsets; + offsets = new long[size]; + long oldLengths[] = lengths; + lengths = new long[size]; + if (preserveData) { + if (isRepeating) { + offsets[0] = oldOffsets[0]; + lengths[0] = oldLengths[0]; + } else { + System.arraycopy(oldOffsets, 0, offsets, 0 , oldOffsets.length); + System.arraycopy(oldLengths, 0, lengths, 0, oldLengths.length); + } + } + } + } + + /** + * Initializee the vector + */ + @Override + public void init() { + super.init(); + childCount = 0; + } + + /** + * Reset the vector for the next batch. + */ + @Override + public void reset() { + super.reset(); + childCount = 0; + } + + @Override + public void shallowCopyTo(ColumnVector otherCv) { + throw new UnsupportedOperationException(); // Implement in future, if needed. + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..45f3ac6d6eb0969c96d461f0cf532f0a783ab61d --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + +/** + * The representation of a vectorized column of struct objects. + * + * Each field is represented by a separate inner ColumnVector. Since this + * ColumnVector doesn't own any per row data other that the isNull flag, the + * isRepeating only covers the isNull array. + */ +public class StructColumnVector extends ColumnVector { + + public ColumnVector[] fields; + + public StructColumnVector() { + this(VectorizedRowBatch.DEFAULT_SIZE); + } + + /** + * Constructor for StructColumnVector + * + * @param len Vector length + * @param fields the field column vectors + */ + public StructColumnVector(int len, ColumnVector... fields) { + super(Type.STRUCT, len); + this.fields = fields; + } + + @Override + public void flatten(boolean selectedInUse, int[] sel, int size) { + flattenPush(); + for(int i=0; i < fields.length; ++i) { + fields[i].flatten(selectedInUse, sel, size); + } + flattenNoNulls(selectedInUse, sel, size); + } + + @Override + public void setElement(int outElementNum, int inputElementNum, + ColumnVector inputVector) { + if (inputVector.isRepeating) { + inputElementNum = 0; + } + if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) { + isNull[outElementNum] = false; + ColumnVector[] inputFields = ((StructColumnVector) inputVector).fields; + for (int i = 0; i < inputFields.length; ++i) { + fields[i].setElement(outElementNum, inputElementNum, inputFields[i]); + } + } else { + noNulls = false; + isNull[outElementNum] = true; + } + } + + @Override + public void stringifyValue(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + buffer.append('['); + for(int i=0; i < fields.length; ++i) { + if (i != 0) { + buffer.append(", "); + } + fields[i].stringifyValue(buffer, row); + } + buffer.append(']'); + } else { + buffer.append("null"); + } + } + + @Override + public void ensureSize(int size, boolean preserveData) { + super.ensureSize(size, preserveData); + for(int i=0; i < fields.length; ++i) { + fields[i].ensureSize(size, preserveData); + } + } + + @Override + public void reset() { + super.reset(); + for(int i =0; i < fields.length; ++i) { + fields[i].reset(); + } + } + + @Override + public void init() { + super.init(); + for(int i =0; i < fields.length; ++i) { + fields[i].init(); + } + } + + @Override + public void unFlatten() { + super.unFlatten(); + for(int i=0; i < fields.length; ++i) { + fields[i].unFlatten(); + } + } + + @Override + public void setRepeating(boolean isRepeating) { + super.setRepeating(isRepeating); + for(int i=0; i < fields.length; ++i) { + fields[i].setRepeating(isRepeating); + } + } + + @Override + public void shallowCopyTo(ColumnVector otherCv) { + throw new UnsupportedOperationException(); // Implement if needed. + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..ef1c817efaac126aab0a35b013aa273c1a70f49f --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java @@ -0,0 +1,427 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec.vector; + +import java.sql.Timestamp; +import java.util.Arrays; + +import org.apache.hadoop.io.Writable; + +/** + * This class represents a nullable timestamp column vector capable of handing a wide range of + * timestamp values. + * + * We store the 2 (value) fields of a Timestamp class in primitive arrays. + * + * We do this to avoid an array of Java Timestamp objects which would have poor storage + * and memory access characteristics. + * + * Generally, the caller will fill in a scratch timestamp object with values from a row, work + * using the scratch timestamp, and then perhaps update the column vector row with a result. + */ +public class TimestampColumnVector extends ColumnVector { + + /* + * The storage arrays for this column vector corresponds to the storage of a Timestamp: + */ + public long[] time; + // The values from Timestamp.getTime(). + + public int[] nanos; + // The values from Timestamp.getNanos(). + + /* + * Scratch objects. + */ + private final Timestamp scratchTimestamp; + + private Writable scratchWritable; + // Supports keeping a TimestampWritable object without having to import that definition... + + /** + * Use this constructor by default. All column vectors + * should normally be the default size. + */ + public TimestampColumnVector() { + this(VectorizedRowBatch.DEFAULT_SIZE); + } + + /** + * Don't use this except for testing purposes. + * + * @param len the number of rows + */ + public TimestampColumnVector(int len) { + super(Type.TIMESTAMP, len); + + time = new long[len]; + nanos = new int[len]; + + scratchTimestamp = new Timestamp(0); + + scratchWritable = null; // Allocated by caller. + } + + /** + * Return the number of rows. + * @return + */ + public int getLength() { + return time.length; + } + + /** + * Return a row's Timestamp.getTime() value. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @return + */ + public long getTime(int elementNum) { + return time[elementNum]; + } + + /** + * Return a row's Timestamp.getNanos() value. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @return + */ + public int getNanos(int elementNum) { + return nanos[elementNum]; + } + + /** + * Set a Timestamp object from a row of the column. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param timestamp + * @param elementNum + */ + public void timestampUpdate(Timestamp timestamp, int elementNum) { + timestamp.setTime(time[elementNum]); + timestamp.setNanos(nanos[elementNum]); + } + + /** + * Return the scratch Timestamp object set from a row. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @return + */ + public Timestamp asScratchTimestamp(int elementNum) { + scratchTimestamp.setTime(time[elementNum]); + scratchTimestamp.setNanos(nanos[elementNum]); + return scratchTimestamp; + } + + /** + * Return the scratch timestamp (contents undefined). + * @return + */ + public Timestamp getScratchTimestamp() { + return scratchTimestamp; + } + + /** + * Return a long representation of a Timestamp. + * @param elementNum + * @return + */ + public long getTimestampAsLong(int elementNum) { + scratchTimestamp.setTime(time[elementNum]); + scratchTimestamp.setNanos(nanos[elementNum]); + return getTimestampAsLong(scratchTimestamp); + } + + /** + * Return a long representation of a Timestamp. + * @param timestamp + * @return + */ + public static long getTimestampAsLong(Timestamp timestamp) { + return millisToSeconds(timestamp.getTime()); + } + + // Copy of TimestampWritable.millisToSeconds + /** + * Rounds the number of milliseconds relative to the epoch down to the nearest whole number of + * seconds. 500 would round to 0, -500 would round to -1. + */ + private static long millisToSeconds(long millis) { + if (millis >= 0) { + return millis / 1000; + } else { + return (millis - 999) / 1000; + } + } + + /** + * Return a double representation of a Timestamp. + * @param elementNum + * @return + */ + public double getDouble(int elementNum) { + scratchTimestamp.setTime(time[elementNum]); + scratchTimestamp.setNanos(nanos[elementNum]); + return getDouble(scratchTimestamp); + } + + /** + * Return a double representation of a Timestamp. + * @param timestamp + * @return + */ + public static double getDouble(Timestamp timestamp) { + // Same algorithm as TimestampWritable (not currently import-able here). + double seconds, nanos; + seconds = millisToSeconds(timestamp.getTime()); + nanos = timestamp.getNanos(); + return seconds + nanos / 1000000000; + } + + /** + * Compare row to Timestamp. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @param timestamp + * @return -1, 0, 1 standard compareTo values. + */ + public int compareTo(int elementNum, Timestamp timestamp) { + return asScratchTimestamp(elementNum).compareTo(timestamp); + } + + /** + * Compare Timestamp to row. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param timestamp + * @param elementNum + * @return -1, 0, 1 standard compareTo values. + */ + public int compareTo(Timestamp timestamp, int elementNum) { + return timestamp.compareTo(asScratchTimestamp(elementNum)); + } + + /** + * Compare a row to another TimestampColumnVector's row. + * @param elementNum1 + * @param timestampColVector2 + * @param elementNum2 + * @return + */ + public int compareTo(int elementNum1, TimestampColumnVector timestampColVector2, + int elementNum2) { + return asScratchTimestamp(elementNum1).compareTo( + timestampColVector2.asScratchTimestamp(elementNum2)); + } + + /** + * Compare another TimestampColumnVector's row to a row. + * @param timestampColVector1 + * @param elementNum1 + * @param elementNum2 + * @return + */ + public int compareTo(TimestampColumnVector timestampColVector1, int elementNum1, + int elementNum2) { + return timestampColVector1.asScratchTimestamp(elementNum1).compareTo( + asScratchTimestamp(elementNum2)); + } + + @Override + public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) { + + TimestampColumnVector timestampColVector = (TimestampColumnVector) inputVector; + + time[outElementNum] = timestampColVector.time[inputElementNum]; + nanos[outElementNum] = timestampColVector.nanos[inputElementNum]; + } + + // Simplify vector by brute-force flattening noNulls and isRepeating + // This can be used to reduce combinatorial explosion of code paths in VectorExpressions + // with many arguments. + public void flatten(boolean selectedInUse, int[] sel, int size) { + flattenPush(); + if (isRepeating) { + isRepeating = false; + long repeatFastTime = time[0]; + int repeatNanos = nanos[0]; + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + time[i] = repeatFastTime; + nanos[i] = repeatNanos; + } + } else { + Arrays.fill(time, 0, size, repeatFastTime); + Arrays.fill(nanos, 0, size, repeatNanos); + } + flattenRepeatingNulls(selectedInUse, sel, size); + } + flattenNoNulls(selectedInUse, sel, size); + } + + /** + * Set a row from a timestamp. + * We assume the entry has already been isRepeated adjusted. + * @param elementNum + * @param timestamp + */ + public void set(int elementNum, Timestamp timestamp) { + if (timestamp == null) { + this.noNulls = false; + this.isNull[elementNum] = true; + } else { + this.time[elementNum] = timestamp.getTime(); + this.nanos[elementNum] = timestamp.getNanos(); + } + } + + /** + * Set a row from the current value in the scratch timestamp. + * @param elementNum + */ + public void setFromScratchTimestamp(int elementNum) { + this.time[elementNum] = scratchTimestamp.getTime(); + this.nanos[elementNum] = scratchTimestamp.getNanos(); + } + + /** + * Set row to standard null value(s). + * We assume the entry has already been isRepeated adjusted. + * @param elementNum + */ + public void setNullValue(int elementNum) { + time[elementNum] = 0; + nanos[elementNum] = 1; + } + + // Copy the current object contents into the output. Only copy selected entries, + // as indicated by selectedInUse and the sel array. + public void copySelected( + boolean selectedInUse, int[] sel, int size, TimestampColumnVector output) { + + // Output has nulls if and only if input has nulls. + output.noNulls = noNulls; + output.isRepeating = false; + + // Handle repeating case + if (isRepeating) { + output.time[0] = time[0]; + output.nanos[0] = nanos[0]; + output.isNull[0] = isNull[0]; + output.isRepeating = true; + return; + } + + // Handle normal case + + // Copy data values over + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.time[i] = time[i]; + output.nanos[i] = nanos[i]; + } + } + else { + System.arraycopy(time, 0, output.time, 0, size); + System.arraycopy(nanos, 0, output.nanos, 0, size); + } + + // Copy nulls over if needed + if (!noNulls) { + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.isNull[i] = isNull[i]; + } + } + else { + System.arraycopy(isNull, 0, output.isNull, 0, size); + } + } + } + + /** + * Fill all the vector entries with a timestamp. + * @param timestamp + */ + public void fill(Timestamp timestamp) { + noNulls = true; + isRepeating = true; + time[0] = timestamp.getTime(); + nanos[0] = timestamp.getNanos(); + } + + /** + * Return a convenience writable object stored by this column vector. + * Supports keeping a TimestampWritable object without having to import that definition... + * @return + */ + public Writable getScratchWritable() { + return scratchWritable; + } + + /** + * Set the convenience writable object stored by this column vector + * @param scratchWritable + */ + public void setScratchWritable(Writable scratchWritable) { + this.scratchWritable = scratchWritable; + } + + @Override + public void stringifyValue(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + scratchTimestamp.setTime(time[row]); + scratchTimestamp.setNanos(nanos[row]); + buffer.append(scratchTimestamp.toString()); + } else { + buffer.append("null"); + } + } + + @Override + public void ensureSize(int size, boolean preserveData) { + super.ensureSize(size, preserveData); + if (size <= time.length) return; + long[] oldTime = time; + int[] oldNanos = nanos; + time = new long[size]; + nanos = new int[size]; + if (preserveData) { + if (isRepeating) { + time[0] = oldTime[0]; + nanos[0] = oldNanos[0]; + } else { + System.arraycopy(oldTime, 0, time, 0, oldTime.length); + System.arraycopy(oldNanos, 0, nanos, 0, oldNanos.length); + } + } + } + + @Override + public void shallowCopyTo(ColumnVector otherCv) { + TimestampColumnVector other = (TimestampColumnVector)otherCv; + super.shallowCopyTo(other); + other.time = time; + other.nanos = nanos; + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..e844b11f9634255f121ab4225e227ef7801adb2e --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + +/** + * The representation of a vectorized column of struct objects. + * + * Each field is represented by a separate inner ColumnVector. Since this + * ColumnVector doesn't own any per row data other that the isNull flag, the + * isRepeating only covers the isNull array. + */ +public class UnionColumnVector extends ColumnVector { + + public int[] tags; + public ColumnVector[] fields; + + public UnionColumnVector() { + this(VectorizedRowBatch.DEFAULT_SIZE); + } + + /** + * Constructor for UnionColumnVector + * + * @param len Vector length + * @param fields the field column vectors + */ + public UnionColumnVector(int len, ColumnVector... fields) { + super(Type.UNION, len); + tags = new int[len]; + this.fields = fields; + } + + @Override + public void flatten(boolean selectedInUse, int[] sel, int size) { + flattenPush(); + for(int i=0; i < fields.length; ++i) { + fields[i].flatten(selectedInUse, sel, size); + } + flattenNoNulls(selectedInUse, sel, size); + } + + @Override + public void setElement(int outElementNum, int inputElementNum, + ColumnVector inputVector) { + if (inputVector.isRepeating) { + inputElementNum = 0; + } + if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) { + isNull[outElementNum] = false; + UnionColumnVector input = (UnionColumnVector) inputVector; + tags[outElementNum] = input.tags[inputElementNum]; + fields[tags[outElementNum]].setElement(outElementNum, inputElementNum, + input.fields[tags[outElementNum]]); + } else { + noNulls = false; + isNull[outElementNum] = true; + } + } + + @Override + public void stringifyValue(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + buffer.append("{\"tag\": "); + buffer.append(tags[row]); + buffer.append(", \"value\": "); + fields[tags[row]].stringifyValue(buffer, row); + buffer.append('}'); + } else { + buffer.append("null"); + } + } + + @Override + public void ensureSize(int size, boolean preserveData) { + super.ensureSize(size, preserveData); + if (tags.length < size) { + if (preserveData) { + int[] oldTags = tags; + tags = new int[size]; + System.arraycopy(oldTags, 0, tags, 0, oldTags.length); + } else { + tags = new int[size]; + } + for(int i=0; i < fields.length; ++i) { + fields[i].ensureSize(size, preserveData); + } + } + } + + @Override + public void reset() { + super.reset(); + for(int i =0; i < fields.length; ++i) { + fields[i].reset(); + } + } + + @Override + public void init() { + super.init(); + for(int i =0; i < fields.length; ++i) { + fields[i].init(); + } + } + + @Override + public void unFlatten() { + super.unFlatten(); + for(int i=0; i < fields.length; ++i) { + fields[i].unFlatten(); + } + } + + @Override + public void setRepeating(boolean isRepeating) { + super.setRepeating(isRepeating); + for(int i=0; i < fields.length; ++i) { + fields[i].setRepeating(isRepeating); + } + } + + @Override + public void shallowCopyTo(ColumnVector otherCv) { + throw new UnsupportedOperationException(); // Implement if needed. + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java new file mode 100644 index 0000000000000000000000000000000000000000..5e5f13ddc7064aa9343376320e50f2da2c3b20ac --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java @@ -0,0 +1,264 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec.vector; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Writable; + +/** + * A VectorizedRowBatch is a set of rows, organized with each column + * as a vector. It is the unit of query execution, organized to minimize + * the cost per row and achieve high cycles-per-instruction. + * The major fields are public by design to allow fast and convenient + * access by the vectorized query execution code. + */ +public class VectorizedRowBatch implements Writable { + public int numCols; // number of columns + public ColumnVector[] cols; // a vector for each column + public int size; // number of rows that qualify (i.e. haven't been filtered out) + public int[] selected; // array of positions of selected values + public int[] projectedColumns; + public int projectionSize; + + private int dataColumnCount; + private int partitionColumnCount; + + + /* + * If no filtering has been applied yet, selectedInUse is false, + * meaning that all rows qualify. If it is true, then the selected[] array + * records the offsets of qualifying rows. + */ + public boolean selectedInUse; + + // If this is true, then there is no data in the batch -- we have hit the end of input. + public boolean endOfFile; + + /* + * This number is carefully chosen to minimize overhead and typically allows + * one VectorizedRowBatch to fit in cache. + */ + public static final int DEFAULT_SIZE = 1024; + + /* + * This number is a safety limit for 32MB of writables. + */ + public static final int DEFAULT_BYTES = 32 * 1024 * 1024; + + /** + * Return a batch with the specified number of columns. + * This is the standard constructor -- all batches should be the same size + * + * @param numCols the number of columns to include in the batch + */ + public VectorizedRowBatch(int numCols) { + this(numCols, DEFAULT_SIZE); + } + + /** + * Return a batch with the specified number of columns and rows. + * Only call this constructor directly for testing purposes. + * Batch size should normally always be defaultSize. + * + * @param numCols the number of columns to include in the batch + * @param size the number of rows to include in the batch + */ + public VectorizedRowBatch(int numCols, int size) { + this.numCols = numCols; + this.size = size; + selected = new int[size]; + selectedInUse = false; + this.cols = new ColumnVector[numCols]; + projectedColumns = new int[numCols]; + + // Initially all columns are projected and in the same order + projectionSize = numCols; + for (int i = 0; i < numCols; i++) { + projectedColumns[i] = i; + } + + dataColumnCount = -1; + partitionColumnCount = -1; + } + + public void setPartitionInfo(int dataColumnCount, int partitionColumnCount) { + this.dataColumnCount = dataColumnCount; + this.partitionColumnCount = partitionColumnCount; + } + + public int getDataColumnCount() { + return dataColumnCount; + } + + public int getPartitionColumnCount() { + return partitionColumnCount; + } + + /** + * Returns the maximum size of the batch (number of rows it can hold) + */ + public int getMaxSize() { + return selected.length; + } + + /** + * Return count of qualifying rows. + * + * @return number of rows that have not been filtered out + */ + public long count() { + return size; + } + + private static String toUTF8(Object o) { + if(o == null || o instanceof NullWritable) { + return "\\N"; /* as found in LazySimpleSerDe's nullSequence */ + } + return o.toString(); + } + + @Override + public String toString() { + if (size == 0) { + return ""; + } + StringBuilder b = new StringBuilder(); + b.append("Column vector types: "); + for (int k = 0; k < projectionSize; k++) { + int projIndex = projectedColumns[k]; + ColumnVector cv = cols[projIndex]; + if (k > 0) { + b.append(", "); + } + b.append(projIndex); + b.append(":"); + String colVectorType = null; + if (cv instanceof LongColumnVector) { + colVectorType = "LONG"; + } else if (cv instanceof DoubleColumnVector) { + colVectorType = "DOUBLE"; + } else if (cv instanceof BytesColumnVector) { + colVectorType = "BYTES"; + } else if (cv instanceof DecimalColumnVector) { + colVectorType = "DECIMAL"; + } else if (cv instanceof TimestampColumnVector) { + colVectorType = "TIMESTAMP"; + } else if (cv instanceof IntervalDayTimeColumnVector) { + colVectorType = "INTERVAL_DAY_TIME"; + } else if (cv instanceof ListColumnVector) { + colVectorType = "LIST"; + } else if (cv instanceof MapColumnVector) { + colVectorType = "MAP"; + } else if (cv instanceof StructColumnVector) { + colVectorType = "STRUCT"; + } else if (cv instanceof UnionColumnVector) { + colVectorType = "UNION"; + } else { + colVectorType = "Unknown"; + } + b.append(colVectorType); + } + b.append('\n'); + + if (this.selectedInUse) { + for (int j = 0; j < size; j++) { + int i = selected[j]; + b.append('['); + for (int k = 0; k < projectionSize; k++) { + int projIndex = projectedColumns[k]; + ColumnVector cv = cols[projIndex]; + if (k > 0) { + b.append(", "); + } + cv.stringifyValue(b, i); + } + b.append(']'); + if (j < size - 1) { + b.append('\n'); + } + } + } else { + for (int i = 0; i < size; i++) { + b.append('['); + for (int k = 0; k < projectionSize; k++) { + int projIndex = projectedColumns[k]; + ColumnVector cv = cols[projIndex]; + if (k > 0) { + b.append(", "); + } + if (cv != null) { + try { + cv.stringifyValue(b, i); + } catch (Exception ex) { + b.append(""); + } + } + } + b.append(']'); + if (i < size - 1) { + b.append('\n'); + } + } + } + return b.toString(); + } + + @Override + public void readFields(DataInput arg0) throws IOException { + throw new UnsupportedOperationException("Do you really need me?"); + } + + @Override + public void write(DataOutput arg0) throws IOException { + throw new UnsupportedOperationException("Don't call me"); + } + + /** + * Resets the row batch to default state + * - sets selectedInUse to false + * - sets size to 0 + * - sets endOfFile to false + * - resets each column + * - inits each column + */ + public void reset() { + selectedInUse = false; + size = 0; + endOfFile = false; + for (ColumnVector vc : cols) { + if (vc != null) { + vc.reset(); + vc.init(); + } + } + } + + /** + * Set the maximum number of rows in the batch. + * Data is not preserved. + */ + public void ensureSize(int rows) { + for(int i=0; i < cols.length; ++i) { + cols[i].ensureSize(rows, false); + } + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java new file mode 100644 index 0000000000000000000000000000000000000000..f2ae9bce709850345307c2068c49bfeb38dd9c44 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java @@ -0,0 +1,414 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import java.util.Arrays; + +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; + +/** + * String expression evaluation helper functions. + */ +public class StringExpr { + + /* Compare two strings from two byte arrays each + * with their own start position and length. + * Use lexicographic unsigned byte value order. + * This is what's used for UTF-8 sort order. + * Return negative value if arg1 < arg2, 0 if arg1 = arg2, + * positive if arg1 > arg2. + */ + public static int compare(byte[] arg1, int start1, int len1, byte[] arg2, int start2, int len2) { + for (int i = 0; i < len1 && i < len2; i++) { + // Note the "& 0xff" is just a way to convert unsigned bytes to signed integer. + int b1 = arg1[i + start1] & 0xff; + int b2 = arg2[i + start2] & 0xff; + if (b1 != b2) { + return b1 - b2; + } + } + return len1 - len2; + } + + /* Determine if two strings are equal from two byte arrays each + * with their own start position and length. + * Use lexicographic unsigned byte value order. + * This is what's used for UTF-8 sort order. + */ + public static boolean equal(byte[] arg1, final int start1, final int len1, + byte[] arg2, final int start2, final int len2) { + if (len1 != len2) { + return false; + } + if (len1 == 0) { + return true; + } + + // do bounds check for OOB exception + if (arg1[start1] != arg2[start2] + || arg1[start1 + len1 - 1] != arg2[start2 + len2 - 1]) { + return false; + } + + if (len1 == len2) { + // prove invariant to the compiler: len1 = len2 + // all array access between (start1, start1+len1) + // and (start2, start2+len2) are valid + // no more OOB exceptions are possible + final int step = 8; + final int remainder = len1 % step; + final int wlen = len1 - remainder; + // suffix first + for (int i = wlen; i < len1; i++) { + if (arg1[start1 + i] != arg2[start2 + i]) { + return false; + } + } + // SIMD loop + for (int i = 0; i < wlen; i += step) { + final int s1 = start1 + i; + final int s2 = start2 + i; + boolean neq = false; + for (int j = 0; j < step; j++) { + neq = (arg1[s1 + j] != arg2[s2 + j]) || neq; + } + if (neq) { + return false; + } + } + } + + return true; + } + + public static int characterCount(byte[] bytes) { + int end = bytes.length; + + // count characters + int j = 0; + int charCount = 0; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + ++charCount; + } + j++; + } + return charCount; + } + + public static int characterCount(byte[] bytes, int start, int length) { + int end = start + length; + + // count characters + int j = start; + int charCount = 0; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + ++charCount; + } + j++; + } + return charCount; + } + + // A setVal with the same function signature as rightTrim, leftTrim, truncate, etc, below. + // Useful for class generation via templates. + public static void assign(BytesColumnVector outV, int i, byte[] bytes, int start, int length) { + // set output vector + outV.setVal(i, bytes, start, length); + } + + /* + * Right trim a slice of a byte array and return the new byte length. + */ + public static int rightTrim(byte[] bytes, int start, int length) { + // skip trailing blank characters + int j = start + length - 1; + while(j >= start && bytes[j] == 0x20) { + j--; + } + + return (j - start) + 1; + } + + /* + * Right trim a slice of a byte array and place the result into element i of a vector. + */ + public static void rightTrim(BytesColumnVector outV, int i, byte[] bytes, int start, int length) { + // skip trailing blank characters + int j = start + length - 1; + while(j >= start && bytes[j] == 0x20) { + j--; + } + + // set output vector + outV.setVal(i, bytes, start, (j - start) + 1); + } + + /* + * Truncate a slice of a byte array to a maximum number of characters and + * return the new byte length. + */ + public static int truncate(byte[] bytes, int start, int length, int maxLength) { + int end = start + length; + + // count characters forward + int j = start; + int charCount = 0; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + if (charCount == maxLength) { + break; + } + ++charCount; + } + j++; + } + return (j - start); + } + + /* + * Truncate a slice of a byte array to a maximum number of characters and + * place the result into element i of a vector. + */ + public static void truncate(BytesColumnVector outV, int i, byte[] bytes, int start, int length, int maxLength) { + int end = start + length; + + // count characters forward + int j = start; + int charCount = 0; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + if (charCount == maxLength) { + break; + } + ++charCount; + } + j++; + } + + // set output vector + outV.setVal(i, bytes, start, (j - start)); + } + + /* + * Truncate a byte array to a maximum number of characters and + * return a byte array with only truncated bytes. + */ + public static byte[] truncateScalar(byte[] bytes, int maxLength) { + int end = bytes.length; + + // count characters forward + int j = 0; + int charCount = 0; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + if (charCount == maxLength) { + break; + } + ++charCount; + } + j++; + } + if (j == end) { + return bytes; + } else { + return Arrays.copyOf(bytes, j); + } + } + + /* + * Right trim and truncate a slice of a byte array to a maximum number of characters and + * return the new byte length. + */ + public static int rightTrimAndTruncate(byte[] bytes, int start, int length, int maxLength) { + int end = start + length; + + // count characters forward and watch for final run of pads + int j = start; + int charCount = 0; + int padRunStart = -1; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + if (charCount == maxLength) { + break; + } + if (bytes[j] == 0x20) { + if (padRunStart == -1) { + padRunStart = j; + } + } else { + padRunStart = -1; + } + ++charCount; + } else { + padRunStart = -1; + } + j++; + } + if (padRunStart != -1) { + return (padRunStart - start); + } else { + return (j - start); + } + } + + /* + * Right trim and truncate a slice of a byte array to a maximum number of characters and + * place the result into element i of a vector. + */ + public static void rightTrimAndTruncate(BytesColumnVector outV, int i, byte[] bytes, int start, int length, int maxLength) { + int end = start + length; + + // count characters forward and watch for final run of pads + int j = start; + int charCount = 0; + int padRunStart = -1; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + if (charCount == maxLength) { + break; + } + if (bytes[j] == 0x20) { + if (padRunStart == -1) { + padRunStart = j; + } + } else { + padRunStart = -1; + } + ++charCount; + } else { + padRunStart = -1; + } + j++; + } + // set output vector + if (padRunStart != -1) { + outV.setVal(i, bytes, start, (padRunStart - start)); + } else { + outV.setVal(i, bytes, start, (j - start) ); + } + } + + /* + * Right trim and truncate a byte array to a maximum number of characters and + * return a byte array with only the trimmed and truncated bytes. + */ + public static byte[] rightTrimAndTruncateScalar(byte[] bytes, int maxLength) { + int end = bytes.length; + + // count characters forward and watch for final run of pads + int j = 0; + int charCount = 0; + int padRunStart = -1; + while(j < end) { + // UTF-8 continuation bytes have 2 high bits equal to 0x80. + if ((bytes[j] & 0xc0) != 0x80) { + if (charCount == maxLength) { + break; + } + if (bytes[j] == 0x20) { + if (padRunStart == -1) { + padRunStart = j; + } + } else { + padRunStart = -1; + } + ++charCount; + } else { + padRunStart = -1; + } + j++; + } + if (padRunStart != -1) { + return Arrays.copyOf(bytes, padRunStart); + } else if (j == end) { + return bytes; + } else { + return Arrays.copyOf(bytes, j); + } + } + + /* + * Compiles the given pattern with a proper algorithm. + */ + public static Finder compile(byte[] pattern) { + return new BoyerMooreHorspool(pattern); + } + + /* + * A finder finds the first index of its pattern in a given byte array. + * Its thread-safety depends on its implementation. + */ + public interface Finder { + int find(byte[] input, int start, int len); + } + + /* + * StringExpr uses Boyer Moore Horspool algorithm to find faster. + * It is thread-safe, because it holds final member instances only. + * See https://en.wikipedia.org/wiki/Boyer–Moore–Horspool_algorithm . + */ + private static class BoyerMooreHorspool implements Finder { + private static final int MAX_BYTE = 0xff; + private final long[] shift = new long[MAX_BYTE]; + private final byte[] pattern; + private final int plen; + + public BoyerMooreHorspool(byte[] pattern) { + this.pattern = pattern; + this.plen = pattern.length; + Arrays.fill(shift, plen); + for (int i = 0; i < plen - 1; i++) { + shift[pattern[i] & MAX_BYTE] = plen - i - 1; + } + } + + public int find(byte[] input, int start, int len) { + if (pattern.length == 0) { + return 0; + } + + final int end = start + len; + int next = start + plen - 1; + final int plen = this.plen; + final byte[] pattern = this.pattern; + while (next < end) { + int s_tmp = next; + int p_tmp = plen - 1; + while (input[s_tmp] == pattern[p_tmp]) { + p_tmp--; + if (p_tmp < 0) { + return s_tmp; + } + s_tmp--; + } + next += shift[input[next] & MAX_BYTE]; + } + return -1; + } + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/ExpressionTree.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/ExpressionTree.java new file mode 100644 index 0000000000000000000000000000000000000000..443083d228502f7c78f3bd4ece320c38812a6c68 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/ExpressionTree.java @@ -0,0 +1,160 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.sarg; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * The inner representation of the SearchArgument. Most users should not + * need this interface, it is only for file formats that need to translate + * the SearchArgument into an internal form. + */ +public class ExpressionTree { + public enum Operator {OR, AND, NOT, LEAF, CONSTANT} + private final Operator operator; + private final List children; + private int leaf; + private final SearchArgument.TruthValue constant; + + ExpressionTree() { + operator = null; + children = null; + leaf = 0; + constant = null; + } + + ExpressionTree(Operator op, ExpressionTree... kids) { + operator = op; + children = new ArrayList(); + leaf = -1; + this.constant = null; + Collections.addAll(children, kids); + } + + ExpressionTree(int leaf) { + operator = Operator.LEAF; + children = null; + this.leaf = leaf; + this.constant = null; + } + + ExpressionTree(SearchArgument.TruthValue constant) { + operator = Operator.CONSTANT; + children = null; + this.leaf = -1; + this.constant = constant; + } + + ExpressionTree(ExpressionTree other) { + this.operator = other.operator; + if (other.children == null) { + this.children = null; + } else { + this.children = new ArrayList(); + for(ExpressionTree child: other.children) { + children.add(new ExpressionTree(child)); + } + } + this.leaf = other.leaf; + this.constant = other.constant; + } + + public SearchArgument.TruthValue evaluate(SearchArgument.TruthValue[] leaves + ) { + SearchArgument.TruthValue result = null; + switch (operator) { + case OR: + for(ExpressionTree child: children) { + result = child.evaluate(leaves).or(result); + } + return result; + case AND: + for(ExpressionTree child: children) { + result = child.evaluate(leaves).and(result); + } + return result; + case NOT: + return children.get(0).evaluate(leaves).not(); + case LEAF: + return leaves[leaf]; + case CONSTANT: + return constant; + default: + throw new IllegalStateException("Unknown operator: " + operator); + } + } + + @Override + public String toString() { + StringBuilder buffer = new StringBuilder(); + switch (operator) { + case OR: + buffer.append("(or"); + for(ExpressionTree child: children) { + buffer.append(' '); + buffer.append(child.toString()); + } + buffer.append(')'); + break; + case AND: + buffer.append("(and"); + for(ExpressionTree child: children) { + buffer.append(' '); + buffer.append(child.toString()); + } + buffer.append(')'); + break; + case NOT: + buffer.append("(not "); + buffer.append(children.get(0)); + buffer.append(')'); + break; + case LEAF: + buffer.append("leaf-"); + buffer.append(leaf); + break; + case CONSTANT: + buffer.append(constant); + break; + } + return buffer.toString(); + } + + public Operator getOperator() { + return operator; + } + + public List getChildren() { + return children; + } + + public SearchArgument.TruthValue getConstant() { + return constant; + } + + public int getLeaf() { + return leaf; + } + + public void setLeaf(int leaf) { + this.leaf = leaf; + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/LiteralDelegate.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/LiteralDelegate.java new file mode 100644 index 0000000000000000000000000000000000000000..bd8a5ce57860d20f08c9012a2332c7dcdec9fee8 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/LiteralDelegate.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.sarg; + +import org.apache.hadoop.conf.Configurable; + +/** + * Interface to retrieve a literal value + */ +public interface LiteralDelegate extends Configurable { + + Object getLiteral(); + + String getId(); +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java new file mode 100644 index 0000000000000000000000000000000000000000..469a3dad47c79e51d24115db469647596e7b9785 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.sarg; + +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; + +import java.sql.Date; +import java.sql.Timestamp; +import java.util.List; + +/** + * The primitive predicates that form a SearchArgument. + */ +public interface PredicateLeaf { + + /** + * The possible operators for predicates. To get the opposites, construct + * an expression with a not operator. + */ + public static enum Operator { + EQUALS, + NULL_SAFE_EQUALS, + LESS_THAN, + LESS_THAN_EQUALS, + IN, + BETWEEN, + IS_NULL + } + + /** + * The possible types for sargs. + */ + public static enum Type { + LONG(Long.class), // all of the integer types + FLOAT(Double.class), // float and double + STRING(String.class), // string, char, varchar + DATE(Date.class), + DECIMAL(HiveDecimalWritable.class), + TIMESTAMP(Timestamp.class), + BOOLEAN(Boolean.class); + + private final Class cls; + Type(Class cls) { + this.cls = cls; + } + + /** + * For all SARG leaves, the values must be the matching class. + * @return the value class + */ + public Class getValueClass() { + return cls; + } + } + + /** + * Get the operator for the leaf. + */ + public Operator getOperator(); + + /** + * Get the type of the column and literal by the file format. + */ + public Type getType(); + + /** + * Get the simple column name. + * @return the column name + */ + public String getColumnName(); + + /** + * Get the literal half of the predicate leaf. Adapt the original type for what orc needs + * + * @return an Integer, Long, Double, or String + */ + public Object getLiteral(); + + /** + * For operators with multiple literals (IN and BETWEEN), get the literals. + * + * @return the list of literals (Integer, Longs, Doubles, or Strings) + * + */ + public List getLiteralList(); +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgument.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgument.java new file mode 100644 index 0000000000000000000000000000000000000000..d70b3b067b54697ee564be6a0189971c20153146 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgument.java @@ -0,0 +1,287 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.sarg; + +import java.util.List; + +/** + * Primary interface for + * SearchArgument, which are the subset of predicates + * that can be pushed down to the RecordReader. Each SearchArgument consists + * of a series of SearchClauses that must each be true for the row to be + * accepted by the filter. + * + * This requires that the filter be normalized into conjunctive normal form + * (CNF). + */ +public interface SearchArgument { + + /** + * The potential result sets of logical operations. + */ + public static enum TruthValue { + YES, NO, NULL, YES_NULL, NO_NULL, YES_NO, YES_NO_NULL; + + /** + * Compute logical or between the two values. + * @param right the other argument or null + * @return the result + */ + public TruthValue or(TruthValue right) { + if (right == null || right == this) { + return this; + } + if (right == YES || this == YES) { + return YES; + } + if (right == YES_NULL || this == YES_NULL) { + return YES_NULL; + } + if (right == NO) { + return this; + } + if (this == NO) { + return right; + } + if (this == NULL) { + if (right == NO_NULL) { + return NULL; + } else { + return YES_NULL; + } + } + if (right == NULL) { + if (this == NO_NULL) { + return NULL; + } else { + return YES_NULL; + } + } + return YES_NO_NULL; + } + + /** + * Compute logical AND between the two values. + * @param right the other argument or null + * @return the result + */ + public TruthValue and(TruthValue right) { + if (right == null || right == this) { + return this; + } + if (right == NO || this == NO) { + return NO; + } + if (right == NO_NULL || this == NO_NULL) { + return NO_NULL; + } + if (right == YES) { + return this; + } + if (this == YES) { + return right; + } + if (this == NULL) { + if (right == YES_NULL) { + return NULL; + } else { + return NO_NULL; + } + } + if (right == NULL) { + if (this == YES_NULL) { + return NULL; + } else { + return NO_NULL; + } + } + return YES_NO_NULL; + } + + public TruthValue not() { + switch (this) { + case NO: + return YES; + case YES: + return NO; + case NULL: + case YES_NO: + case YES_NO_NULL: + return this; + case NO_NULL: + return YES_NULL; + case YES_NULL: + return NO_NULL; + default: + throw new IllegalArgumentException("Unknown value: " + this); + } + } + + /** + * Does the RecordReader need to include this set of records? + * @return true unless none of the rows qualify + */ + public boolean isNeeded() { + switch (this) { + case NO: + case NULL: + case NO_NULL: + return false; + default: + return true; + } + } + } + + /** + * Get the leaf predicates that are required to evaluate the predicate. The + * list will have the duplicates removed. + * @return the list of leaf predicates + */ + public List getLeaves(); + + /** + * Get the expression tree. This should only needed for file formats that + * need to translate the expression to an internal form. + */ + public ExpressionTree getExpression(); + + /** + * Evaluate the entire predicate based on the values for the leaf predicates. + * @param leaves the value of each leaf predicate + * @return the value of hte entire predicate + */ + public TruthValue evaluate(TruthValue[] leaves); + + /** + * A builder object for contexts outside of Hive where it isn't easy to + * get a ExprNodeDesc. The user must call startOr, startAnd, or startNot + * before adding any leaves. + */ + public interface Builder { + + /** + * Start building an or operation and push it on the stack. + * @return this + */ + public Builder startOr(); + + /** + * Start building an and operation and push it on the stack. + * @return this + */ + public Builder startAnd(); + + /** + * Start building a not operation and push it on the stack. + * @return this + */ + public Builder startNot(); + + /** + * Finish the current operation and pop it off of the stack. Each start + * call must have a matching end. + * @return this + */ + public Builder end(); + + /** + * Add a less than leaf to the current item on the stack. + * @param column the name of the column + * @param type the type of the expression + * @param literal the literal + * @return this + */ + public Builder lessThan(String column, PredicateLeaf.Type type, + Object literal); + + /** + * Add a less than equals leaf to the current item on the stack. + * @param column the name of the column + * @param type the type of the expression + * @param literal the literal + * @return this + */ + public Builder lessThanEquals(String column, PredicateLeaf.Type type, + Object literal); + + /** + * Add an equals leaf to the current item on the stack. + * @param column the name of the column + * @param type the type of the expression + * @param literal the literal + * @return this + */ + public Builder equals(String column, PredicateLeaf.Type type, + Object literal); + + /** + * Add a null safe equals leaf to the current item on the stack. + * @param column the name of the column + * @param type the type of the expression + * @param literal the literal + * @return this + */ + public Builder nullSafeEquals(String column, PredicateLeaf.Type type, + Object literal); + + /** + * Add an in leaf to the current item on the stack. + * @param column the name of the column + * @param type the type of the expression + * @param literal the literal + * @return this + */ + public Builder in(String column, PredicateLeaf.Type type, + Object... literal); + + /** + * Add an is null leaf to the current item on the stack. + * @param column the name of the column + * @param type the type of the expression + * @return this + */ + public Builder isNull(String column, PredicateLeaf.Type type); + + /** + * Add a between leaf to the current item on the stack. + * @param column the name of the column + * @param type the type of the expression + * @param lower the literal + * @param upper the literal + * @return this + */ + public Builder between(String column, PredicateLeaf.Type type, + Object lower, Object upper); + + /** + * Add a truth value to the expression. + * @param truth + * @return this + */ + public Builder literal(TruthValue truth); + + /** + * Build and return the SearchArgument that has been defined. All of the + * starts must have been ended before this call. + * @return the new SearchArgument + */ + public SearchArgument build(); + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentFactory.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..3c10c83644587def21f8f5d7677ccf9795f75426 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentFactory.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.sarg; + +import org.apache.hadoop.conf.Configuration; + +/** + * A factory for creating SearchArguments, as well as modifying those created by this factory. + */ +public class SearchArgumentFactory { + public static SearchArgument.Builder newBuilder() { + return newBuilder(null); + } + + public static SearchArgument.Builder newBuilder(Configuration conf) { + return new SearchArgumentImpl.BuilderImpl(conf); + } + + public static void setPredicateLeafColumn(PredicateLeaf leaf, String newName) { + SearchArgumentImpl.PredicateLeafImpl.setColumnName(leaf, newName); + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..85273f9c161f5db190987f63cb9437aa6bd8b8a6 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java @@ -0,0 +1,739 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.sarg; + +import java.sql.Timestamp; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; + +/** + * The implementation of SearchArguments. Visible for testing only. + */ +public final class SearchArgumentImpl implements SearchArgument { + + public static final class PredicateLeafImpl implements PredicateLeaf { + private final Operator operator; + private final Type type; + private String columnName; + private final Object literal; + private final List literalList; + + // Used by kryo + @SuppressWarnings("unused") + PredicateLeafImpl() { + operator = null; + type = null; + columnName = null; + literal = null; + literalList = null; + } + + public PredicateLeafImpl(Operator operator, + Type type, + String columnName, + Object literal, + List literalList) { + this(operator, type, columnName, literal, literalList, null); + } + + public PredicateLeafImpl(Operator operator, + Type type, + String columnName, + Object literal, + List literalList, Configuration conf) { + this.operator = operator; + this.type = type; + this.columnName = columnName; + this.literal = literal; + checkLiteralType(literal, type, conf); + this.literalList = literalList; + if (literalList != null) { + Class valueCls = type.getValueClass(); + for(Object lit: literalList) { + checkLiteralType(lit, type, conf); + } + } + } + + @Override + public Operator getOperator() { + return operator; + } + + @Override + public Type getType(){ + return type; + } + + @Override + public String getColumnName() { + return columnName; + } + + @Override + public Object getLiteral() { + if (literal instanceof LiteralDelegate) { + return ((LiteralDelegate) literal).getLiteral(); + } + + // To get around a kryo 2.22 bug while deserialize a Timestamp into Date + // (https://github.com/EsotericSoftware/kryo/issues/88) + // When we see a Date, convert back into Timestamp + if (literal instanceof java.util.Date) { + return new Timestamp(((java.util.Date)literal).getTime()); + } + return literal; + } + + @Override + public List getLiteralList() { + if (literalList != null && literalList.size() > 0 && literalList.get(0) instanceof LiteralDelegate) { + List newLiteraList = new ArrayList(); + for (Object litertalObj : literalList) { + Object literal = ((LiteralDelegate) litertalObj).getLiteral(); + if (literal != null) { + newLiteraList.add(literal); + } + } + return newLiteraList; + } + return literalList; + } + + @Override + public String toString() { + StringBuilder buffer = new StringBuilder(); + buffer.append('('); + buffer.append(operator); + buffer.append(' '); + buffer.append(columnName); + if (literal != null) { + buffer.append(' '); + buffer.append(literal); + } else if (literalList != null) { + for(Object lit: literalList) { + buffer.append(' '); + buffer.append(lit == null ? "null" : lit.toString()); + } + } + buffer.append(')'); + return buffer.toString(); + } + + private static boolean isEqual(Object left, Object right) { + + return left == right || + (left != null && right != null && left.equals(right)); + } + + @Override + public boolean equals(Object other) { + if (other == null || other.getClass() != getClass()) { + return false; + } else if (other == this) { + return true; + } else { + PredicateLeafImpl o = (PredicateLeafImpl) other; + return operator == o.operator && + type == o.type && + columnName.equals(o.columnName) && + isEqual(literal, o.literal) && + isEqual(literalList, o.literalList); + } + } + + @Override + public int hashCode() { + return operator.hashCode() + + type.hashCode() * 17 + + columnName.hashCode() * 3 * 17+ + (literal == null ? 0 : literal.hashCode()) * 101 * 3 * 17 + + (literalList == null ? 0 : literalList.hashCode()) * + 103 * 101 * 3 * 17; + } + + public static void setColumnName(PredicateLeaf leaf, String newName) { + assert leaf instanceof PredicateLeafImpl; + ((PredicateLeafImpl)leaf).columnName = newName; + } + + protected void checkLiteralType(Object literal, Type type, Configuration conf) { + if (literal == null) { + return; + } + + if (literal instanceof LiteralDelegate) { + // Give it a pass. Optionally, have LiteralDelegate provide a getLiteralClass() to check. + ((LiteralDelegate) literal).setConf(conf); + } else { + if (literal.getClass() != type.getValueClass()) { + throw new IllegalArgumentException("Wrong value class " + + literal.getClass().getName() + " for " + type + "." + operator + + " leaf"); + } + } + } + } + + private final List leaves; + private final ExpressionTree expression; + + SearchArgumentImpl(ExpressionTree expression, List leaves) { + this.expression = expression; + this.leaves = leaves; + } + + // Used by kyro + @SuppressWarnings("unused") + SearchArgumentImpl() { + leaves = null; + expression = null; + } + + @Override + public List getLeaves() { + return leaves; + } + + @Override + public TruthValue evaluate(TruthValue[] leaves) { + return expression == null ? TruthValue.YES : expression.evaluate(leaves); + } + + @Override + public ExpressionTree getExpression() { + return expression; + } + + @Override + public String toString() { + StringBuilder buffer = new StringBuilder(); + for(int i=0; i < leaves.size(); ++i) { + buffer.append("leaf-"); + buffer.append(i); + buffer.append(" = "); + buffer.append(leaves.get(i).toString()); + buffer.append(", "); + } + buffer.append("expr = "); + buffer.append(expression); + return buffer.toString(); + } + + static class BuilderImpl implements Builder { + + Configuration conf; + public BuilderImpl(Configuration conf) { + this.conf = conf; + } + + // max threshold for CNF conversion. having >8 elements in andList will be + // converted to maybe + private static final int CNF_COMBINATIONS_THRESHOLD = 256; + + private final Deque currentTree = + new ArrayDeque(); + private final Map leaves = + new HashMap(); + private final ExpressionTree root = + new ExpressionTree(ExpressionTree.Operator.AND); + { + currentTree.add(root); + } + + @Override + public Builder startOr() { + ExpressionTree node = new ExpressionTree(ExpressionTree.Operator.OR); + currentTree.getFirst().getChildren().add(node); + currentTree.addFirst(node); + return this; + } + + @Override + public Builder startAnd() { + ExpressionTree node = new ExpressionTree(ExpressionTree.Operator.AND); + currentTree.getFirst().getChildren().add(node); + currentTree.addFirst(node); + return this; + } + + @Override + public Builder startNot() { + ExpressionTree node = new ExpressionTree(ExpressionTree.Operator.NOT); + currentTree.getFirst().getChildren().add(node); + currentTree.addFirst(node); + return this; + } + + @Override + public Builder end() { + ExpressionTree current = currentTree.removeFirst(); + if (current.getChildren().size() == 0) { + throw new IllegalArgumentException("Can't create expression " + root + + " with no children."); + } + if (current.getOperator() == ExpressionTree.Operator.NOT && + current.getChildren().size() != 1) { + throw new IllegalArgumentException("Can't create not expression " + + current + " with more than 1 child."); + } + return this; + } + + private int addLeaf(PredicateLeaf leaf) { + Integer result = leaves.get(leaf); + if (result == null) { + int id = leaves.size(); + leaves.put(leaf, id); + return id; + } else { + return result; + } + } + + @Override + public Builder lessThan(String column, PredicateLeaf.Type type, + Object literal) { + ExpressionTree parent = currentTree.getFirst(); + if (column == null || literal == null) { + parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL)); + } else { + PredicateLeaf leaf = + new PredicateLeafImpl(PredicateLeaf.Operator.LESS_THAN, + type, column, literal, null, conf); + parent.getChildren().add(new ExpressionTree(addLeaf(leaf))); + } + return this; + } + + @Override + public Builder lessThanEquals(String column, PredicateLeaf.Type type, + Object literal) { + ExpressionTree parent = currentTree.getFirst(); + if (column == null || literal == null) { + parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL)); + } else { + PredicateLeaf leaf = + new PredicateLeafImpl(PredicateLeaf.Operator.LESS_THAN_EQUALS, + type, column, literal, null, conf); + parent.getChildren().add(new ExpressionTree(addLeaf(leaf))); + } + return this; + } + + @Override + public Builder equals(String column, PredicateLeaf.Type type, + Object literal) { + ExpressionTree parent = currentTree.getFirst(); + if (column == null || literal == null) { + parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL)); + } else { + PredicateLeaf leaf = + new PredicateLeafImpl(PredicateLeaf.Operator.EQUALS, + type, column, literal, null, conf); + parent.getChildren().add(new ExpressionTree(addLeaf(leaf))); + } + return this; + } + + @Override + public Builder nullSafeEquals(String column, PredicateLeaf.Type type, + Object literal) { + ExpressionTree parent = currentTree.getFirst(); + if (column == null || literal == null) { + parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL)); + } else { + PredicateLeaf leaf = + new PredicateLeafImpl(PredicateLeaf.Operator.NULL_SAFE_EQUALS, + type, column, literal, null, conf); + parent.getChildren().add(new ExpressionTree(addLeaf(leaf))); + } + return this; + } + + @Override + public Builder in(String column, PredicateLeaf.Type type, + Object... literal) { + ExpressionTree parent = currentTree.getFirst(); + if (column == null || literal == null) { + parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL)); + } else { + if (literal.length == 0) { + throw new IllegalArgumentException("Can't create in expression with " + + "no arguments"); + } + List argList = new ArrayList(); + argList.addAll(Arrays.asList(literal)); + + PredicateLeaf leaf = + new PredicateLeafImpl(PredicateLeaf.Operator.IN, + type, column, null, argList, conf); + parent.getChildren().add(new ExpressionTree(addLeaf(leaf))); + } + return this; + } + + @Override + public Builder isNull(String column, PredicateLeaf.Type type) { + ExpressionTree parent = currentTree.getFirst(); + if (column == null) { + parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL)); + } else { + PredicateLeaf leaf = + new PredicateLeafImpl(PredicateLeaf.Operator.IS_NULL, + type, column, null, null, conf); + parent.getChildren().add(new ExpressionTree(addLeaf(leaf))); + } + return this; + } + + @Override + public Builder between(String column, PredicateLeaf.Type type, Object lower, + Object upper) { + ExpressionTree parent = currentTree.getFirst(); + if (column == null || lower == null || upper == null) { + parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL)); + } else { + List argList = new ArrayList(); + argList.add(lower); + argList.add(upper); + PredicateLeaf leaf = + new PredicateLeafImpl(PredicateLeaf.Operator.BETWEEN, + type, column, null, argList, conf); + parent.getChildren().add(new ExpressionTree(addLeaf(leaf))); + } + return this; + } + + @Override + public Builder literal(TruthValue truth) { + ExpressionTree parent = currentTree.getFirst(); + parent.getChildren().add(new ExpressionTree(truth)); + return this; + } + + /** + * Recursively explore the tree to find the leaves that are still reachable + * after optimizations. + * @param tree the node to check next + * @param next the next available leaf id + * @param leafReorder + * @return the next available leaf id + */ + static int compactLeaves(ExpressionTree tree, int next, int[] leafReorder) { + if (tree.getOperator() == ExpressionTree.Operator.LEAF) { + int oldLeaf = tree.getLeaf(); + if (leafReorder[oldLeaf] == -1) { + leafReorder[oldLeaf] = next++; + } + } else if (tree.getChildren() != null){ + for(ExpressionTree child: tree.getChildren()) { + next = compactLeaves(child, next, leafReorder); + } + } + return next; + } + + /** + * Rewrite expression tree to update the leaves. + * @param root the root of the tree to fix + * @param leafReorder a map from old leaf ids to new leaf ids + * @return the fixed root + */ + static ExpressionTree rewriteLeaves(ExpressionTree root, + int[] leafReorder) { + // The leaves could be shared in the tree. Use Set to remove the duplicates. + Set leaves = new HashSet(); + Queue nodes = new LinkedList(); + nodes.add(root); + + while(!nodes.isEmpty()) { + ExpressionTree node = nodes.remove(); + if (node.getOperator() == ExpressionTree.Operator.LEAF) { + leaves.add(node); + } else { + if (node.getChildren() != null){ + nodes.addAll(node.getChildren()); + } + } + } + + // Update the leaf in place + for(ExpressionTree leaf : leaves) { + leaf.setLeaf(leafReorder[leaf.getLeaf()]); + } + + return root; + } + + @Override + public SearchArgument build() { + if (currentTree.size() != 1) { + throw new IllegalArgumentException("Failed to end " + + currentTree.size() + " operations."); + } + ExpressionTree optimized = pushDownNot(root); + optimized = foldMaybe(optimized); + optimized = flatten(optimized); + optimized = convertToCNF(optimized); + optimized = flatten(optimized); + int leafReorder[] = new int[leaves.size()]; + Arrays.fill(leafReorder, -1); + int newLeafCount = compactLeaves(optimized, 0, leafReorder); + optimized = rewriteLeaves(optimized, leafReorder); + ArrayList leafList = new ArrayList<>(newLeafCount); + // expand list to correct size + for(int i=0; i < newLeafCount; ++i) { + leafList.add(null); + } + // build the new list + for(Map.Entry elem: leaves.entrySet()) { + int newLoc = leafReorder[elem.getValue()]; + if (newLoc != -1) { + leafList.set(newLoc, elem.getKey()); + } + } + return new SearchArgumentImpl(optimized, leafList); + } + + /** + * Push the negations all the way to just before the leaves. Also remove + * double negatives. + * @param root the expression to normalize + * @return the normalized expression, which may share some or all of the + * nodes of the original expression. + */ + static ExpressionTree pushDownNot(ExpressionTree root) { + if (root.getOperator() == ExpressionTree.Operator.NOT) { + ExpressionTree child = root.getChildren().get(0); + switch (child.getOperator()) { + case NOT: + return pushDownNot(child.getChildren().get(0)); + case CONSTANT: + return new ExpressionTree(child.getConstant().not()); + case AND: + root = new ExpressionTree(ExpressionTree.Operator.OR); + for(ExpressionTree kid: child.getChildren()) { + root.getChildren().add(pushDownNot(new + ExpressionTree(ExpressionTree.Operator.NOT, kid))); + } + break; + case OR: + root = new ExpressionTree(ExpressionTree.Operator.AND); + for(ExpressionTree kid: child.getChildren()) { + root.getChildren().add(pushDownNot(new ExpressionTree + (ExpressionTree.Operator.NOT, kid))); + } + break; + // for leaf, we don't do anything + default: + break; + } + } else if (root.getChildren() != null) { + // iterate through children and push down not for each one + for(int i=0; i < root.getChildren().size(); ++i) { + root.getChildren().set(i, pushDownNot(root.getChildren().get(i))); + } + } + return root; + } + + /** + * Remove MAYBE values from the expression. If they are in an AND operator, + * they are dropped. If they are in an OR operator, they kill their parent. + * This assumes that pushDownNot has already been called. + * @param expr The expression to clean up + * @return The cleaned up expression + */ + static ExpressionTree foldMaybe(ExpressionTree expr) { + if (expr.getChildren() != null) { + for(int i=0; i < expr.getChildren().size(); ++i) { + ExpressionTree child = foldMaybe(expr.getChildren().get(i)); + if (child.getConstant() == TruthValue.YES_NO_NULL) { + switch (expr.getOperator()) { + case AND: + expr.getChildren().remove(i); + i -= 1; + break; + case OR: + // a maybe will kill the or condition + return child; + default: + throw new IllegalStateException("Got a maybe as child of " + + expr); + } + } else { + expr.getChildren().set(i, child); + } + } + if (expr.getChildren().isEmpty()) { + return new ExpressionTree(TruthValue.YES_NO_NULL); + } + } + return expr; + } + + /** + * Converts multi-level ands and ors into single level ones. + * @param root the expression to flatten + * @return the flattened expression, which will always be root with + * potentially modified children. + */ + static ExpressionTree flatten(ExpressionTree root) { + if (root.getChildren() != null) { + // iterate through the index, so that if we add more children, + // they don't get re-visited + for(int i=0; i < root.getChildren().size(); ++i) { + ExpressionTree child = flatten(root.getChildren().get(i)); + // do we need to flatten? + if (child.getOperator() == root.getOperator() && + child.getOperator() != ExpressionTree.Operator.NOT) { + boolean first = true; + for(ExpressionTree grandkid: child.getChildren()) { + // for the first grandkid replace the original parent + if (first) { + first = false; + root.getChildren().set(i, grandkid); + } else { + root.getChildren().add(++i, grandkid); + } + } + } else { + root.getChildren().set(i, child); + } + } + // if we have a singleton AND or OR, just return the child + if ((root.getOperator() == ExpressionTree.Operator.OR || + root.getOperator() == ExpressionTree.Operator.AND) && + root.getChildren().size() == 1) { + return root.getChildren().get(0); + } + } + return root; + } + + /** + * Generate all combinations of items on the andList. For each item on the + * andList, it generates all combinations of one child from each and + * expression. Thus, (and a b) (and c d) will be expanded to: (or a c) + * (or a d) (or b c) (or b d). If there are items on the nonAndList, they + * are added to each or expression. + * @param result a list to put the results onto + * @param andList a list of and expressions + * @param nonAndList a list of non-and expressions + */ + private static void generateAllCombinations(List result, + List andList, + List nonAndList + ) { + List kids = andList.get(0).getChildren(); + if (result.isEmpty()) { + for(ExpressionTree kid: kids) { + ExpressionTree or = new ExpressionTree(ExpressionTree.Operator.OR); + result.add(or); + for(ExpressionTree node: nonAndList) { + or.getChildren().add(new ExpressionTree(node)); + } + or.getChildren().add(kid); + } + } else { + List work = new ArrayList(result); + result.clear(); + for(ExpressionTree kid: kids) { + for(ExpressionTree or: work) { + ExpressionTree copy = new ExpressionTree(or); + copy.getChildren().add(kid); + result.add(copy); + } + } + } + if (andList.size() > 1) { + generateAllCombinations(result, andList.subList(1, andList.size()), + nonAndList); + } + } + + /** + * Convert an expression so that the top level operator is AND with OR + * operators under it. This routine assumes that all of the NOT operators + * have been pushed to the leaves via pushdDownNot. + * @param root the expression + * @return the normalized expression + */ + static ExpressionTree convertToCNF(ExpressionTree root) { + if (root.getChildren() != null) { + // convert all of the children to CNF + int size = root.getChildren().size(); + for(int i=0; i < size; ++i) { + root.getChildren().set(i, convertToCNF(root.getChildren().get(i))); + } + if (root.getOperator() == ExpressionTree.Operator.OR) { + // a list of leaves that weren't under AND expressions + List nonAndList = new ArrayList(); + // a list of AND expressions that we need to distribute + List andList = new ArrayList(); + for(ExpressionTree child: root.getChildren()) { + if (child.getOperator() == ExpressionTree.Operator.AND) { + andList.add(child); + } else if (child.getOperator() == ExpressionTree.Operator.OR) { + // pull apart the kids of the OR expression + for(ExpressionTree grandkid: child.getChildren()) { + nonAndList.add(grandkid); + } + } else { + nonAndList.add(child); + } + } + if (!andList.isEmpty()) { + if (checkCombinationsThreshold(andList)) { + root = new ExpressionTree(ExpressionTree.Operator.AND); + generateAllCombinations(root.getChildren(), andList, nonAndList); + } else { + root = new ExpressionTree(TruthValue.YES_NO_NULL); + } + } + } + } + return root; + } + + private static boolean checkCombinationsThreshold(List andList) { + int numComb = 1; + for (ExpressionTree tree : andList) { + numComb *= tree.getChildren().size(); + if (numComb > CNF_COMBINATIONS_THRESHOLD) { + return false; + } + } + return true; + } + + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java new file mode 100644 index 0000000000000000000000000000000000000000..68ea6dba41da74e324cc0cfb8026153062be2d3d --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java @@ -0,0 +1,345 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Estimation of memory footprint of object + */ +public enum JavaDataModel { + + JAVA32 { + @Override + public int object() { + return JAVA32_OBJECT; + } + + @Override + public int array() { + return JAVA32_ARRAY; + } + + @Override + public int ref() { + return JAVA32_REF; + } + + @Override + public int hashMap(int entry) { + // base = JAVA32_OBJECT + PRIMITIVES1 * 4 + JAVA32_FIELDREF * 3 + JAVA32_ARRAY; + // entry = JAVA32_OBJECT + JAVA32_FIELDREF + PRIMITIVES1 + return hashMapBase() + hashMapEntry() * entry; + } + + @Override + public int hashMapBase() { + return 64; + } + + @Override + public int hashMapEntry() { + return 24; + } + + @Override + public int hashSet(int entry) { + // hashMap += JAVA32_OBJECT + return hashSetBase() + hashSetEntry() * entry; + } + + @Override + public int hashSetBase() { + return 80; + } + + @Override + public int hashSetEntry() { + return 24; + } + + @Override + public int linkedHashMap(int entry) { + // hashMap += JAVA32_FIELDREF + PRIMITIVES1 + // hashMap.entry += JAVA32_FIELDREF * 2 + return 72 + 32 * entry; + } + + @Override + public int linkedList(int entry) { + // base = JAVA32_OBJECT + PRIMITIVES1 * 2 + JAVA32_FIELDREF; + // entry = JAVA32_OBJECT + JAVA32_FIELDREF * 2 + return linkedListBase() + linkedListEntry() * entry; + } + + @Override + public int linkedListBase() { + return 28; + } + + @Override + public int linkedListEntry() { + return 24; + } + + @Override + public int arrayList() { + // JAVA32_OBJECT + PRIMITIVES1 * 2 + JAVA32_ARRAY; + return 44; + } + + @Override + public int memoryAlign() { + return 8; + } + }, JAVA64 { + @Override + public int object() { + return JAVA64_OBJECT; + } + + @Override + public int array() { + return JAVA64_ARRAY; + } + + @Override + public int ref() { + return JAVA64_REF; + } + + @Override + public int hashMap(int entry) { + // base = JAVA64_OBJECT + PRIMITIVES1 * 4 + JAVA64_FIELDREF * 3 + JAVA64_ARRAY; + // entry = JAVA64_OBJECT + JAVA64_FIELDREF + PRIMITIVES1 + return hashMapBase() + hashMapEntry() * entry; + } + + @Override + public int hashMapBase() { + return 112; + } + + + @Override + public int hashMapEntry() { + return 44; + } + + @Override + public int hashSet(int entry) { + // hashMap += JAVA64_OBJECT + return hashSetBase() + hashSetEntry() * entry; + } + + @Override + public int hashSetBase() { + return 144; + } + + @Override + public int hashSetEntry() { + return 44; + } + + @Override + public int linkedHashMap(int entry) { + // hashMap += JAVA64_FIELDREF + PRIMITIVES1 + // hashMap.entry += JAVA64_FIELDREF * 2 + return 128 + 60 * entry; + } + + @Override + public int linkedList(int entry) { + // base = JAVA64_OBJECT + PRIMITIVES1 * 2 + JAVA64_FIELDREF; + // entry = JAVA64_OBJECT + JAVA64_FIELDREF * 2 + return linkedListBase() + linkedListEntry() * entry; + } + + @Override + public int linkedListBase() { + return 48; + } + + @Override + public int linkedListEntry() { + return 48; + } + + @Override + public int arrayList() { + // JAVA64_OBJECT + PRIMITIVES1 * 2 + JAVA64_ARRAY; + return 80; + } + + @Override + public int memoryAlign() { + return 8; + } + }; + + public abstract int object(); + public abstract int array(); + public abstract int ref(); + public abstract int hashMap(int entry); + public abstract int hashMapBase(); + public abstract int hashMapEntry(); + public abstract int hashSetBase(); + public abstract int hashSetEntry(); + public abstract int hashSet(int entry); + public abstract int linkedHashMap(int entry); + public abstract int linkedListBase(); + public abstract int linkedListEntry(); + public abstract int linkedList(int entry); + public abstract int arrayList(); + public abstract int memoryAlign(); + + // ascii string + public long lengthFor(String string) { + return lengthForStringOfLength(string.length()); + } + + public int lengthForRandom() { + // boolean + double + AtomicLong + return object() + primitive1() + primitive2() + object() + primitive2(); + } + + public int primitive1() { + return PRIMITIVES1; + } + public int primitive2() { + return PRIMITIVES2; + } + + public static long alignUp(long value, long align) { + return (value + align - 1L) & ~(align - 1L); + } + + private static final Logger LOG = LoggerFactory.getLogger(JavaDataModel.class); + + public static final int JAVA32_META = 12; + public static final int JAVA32_ARRAY_META = 16; + public static final int JAVA32_REF = 4; + public static final int JAVA32_OBJECT = 16; // JAVA32_META + JAVA32_REF + public static final int JAVA32_ARRAY = 20; // JAVA32_ARRAY_META + JAVA32_REF + + public static final int JAVA64_META = 24; + public static final int JAVA64_ARRAY_META = 32; + public static final int JAVA64_REF = 8; + public static final int JAVA64_OBJECT = 32; // JAVA64_META + JAVA64_REF + public static final int JAVA64_ARRAY = 40; // JAVA64_ARRAY_META + JAVA64_REF + + public static final int PRIMITIVES1 = 4; // void, boolean, byte, short, int, float + public static final int PRIMITIVES2 = 8; // long, double + + public static final int PRIMITIVE_BYTE = 1; // byte + + private static final class LazyHolder { + private static final JavaDataModel MODEL_FOR_SYSTEM = getModelForSystem(); + } + + //@VisibleForTesting + static JavaDataModel getModelForSystem() { + String props = null; + try { + props = System.getProperty("sun.arch.data.model"); + } catch (Exception e) { + LOG.warn("Failed to determine java data model, defaulting to 64", e); + } + if ("32".equals(props)) { + return JAVA32; + } + // TODO: separate model is needed for compressedOops, which can be guessed from memory size. + return JAVA64; + } + + public static JavaDataModel get() { + return LazyHolder.MODEL_FOR_SYSTEM; + } + + public static int round(int size) { + JavaDataModel model = get(); + if (model == JAVA32 || size % 8 == 0) { + return size; + } + return ((size + 8) >> 3) << 3; + } + + public long lengthForPrimitiveArrayOfSize(int primitiveSize, long length) { + return alignUp(array() + primitiveSize*length, memoryAlign()); + } + + public long lengthForByteArrayOfSize(long length) { + return lengthForPrimitiveArrayOfSize(PRIMITIVE_BYTE, length); + } + public long lengthForObjectArrayOfSize(long length) { + return lengthForPrimitiveArrayOfSize(ref(), length); + } + public long lengthForLongArrayOfSize(long length) { + return lengthForPrimitiveArrayOfSize(primitive2(), length); + } + public long lengthForDoubleArrayOfSize(long length) { + return lengthForPrimitiveArrayOfSize(primitive2(), length); + } + public long lengthForIntArrayOfSize(long length) { + return lengthForPrimitiveArrayOfSize(primitive1(), length); + } + public long lengthForBooleanArrayOfSize(long length) { + return lengthForPrimitiveArrayOfSize(PRIMITIVE_BYTE, length); + } + public long lengthForTimestampArrayOfSize(long length) { + return lengthForPrimitiveArrayOfSize(lengthOfTimestamp(), length); + } + public long lengthForDateArrayOfSize(long length) { + return lengthForPrimitiveArrayOfSize(lengthOfDate(), length); + } + public long lengthForDecimalArrayOfSize(long length) { + return lengthForPrimitiveArrayOfSize(lengthOfDecimal(), length); + } + + public int lengthOfDecimal() { + // object overhead + 8 bytes for intCompact + 4 bytes for precision + // + 4 bytes for scale + size of BigInteger + return object() + 2 * primitive2() + lengthOfBigInteger(); + } + + private int lengthOfBigInteger() { + // object overhead + 4 bytes for bitCount + 4 bytes for bitLength + // + 4 bytes for firstNonzeroByteNum + 4 bytes for firstNonzeroIntNum + + // + 4 bytes for lowestSetBit + 5 bytes for size of magnitude (since max precision + // is only 38 for HiveDecimal) + 7 bytes of padding (since java memory allocations + // are 8 byte aligned) + return object() + 4 * primitive2(); + } + + public int lengthOfTimestamp() { + // object overhead + 4 bytes for int (nanos) + 4 bytes of padding + return object() + primitive2(); + } + + public int lengthOfDate() { + // object overhead + 8 bytes for long (fastTime) + 16 bytes for cdate + return object() + 3 * primitive2(); + } + + public int lengthForStringOfLength(int strLen) { + return object() + primitive1() * 3 + array() + strLen; + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..dfc72728bf5a71bef3686880035a30843e8df23a --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.util; + +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveDecimalV1; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; + +import java.math.BigDecimal; +import java.sql.Timestamp; + +/** + * Utitilities for Timestamps and the relevant conversions. + */ +public class TimestampUtils { + public static final BigDecimal BILLION_BIG_DECIMAL = BigDecimal.valueOf(1000000000); + + /** + * Convert the timestamp to a double measured in seconds. + * @return double representation of the timestamp, accurate to nanoseconds + */ + public static double getDouble(Timestamp ts) { + long seconds = millisToSeconds(ts.getTime()); + return seconds + ((double) ts.getNanos()) / 1000000000; + } + + public static Timestamp doubleToTimestamp(double f) { + try { + long seconds = (long) f; + + // We must ensure the exactness of the double's fractional portion. + // 0.6 as the fraction part will be converted to 0.59999... and + // significantly reduce the savings from binary serialization + BigDecimal bd = new BigDecimal(String.valueOf(f)); + + bd = bd.subtract(new BigDecimal(seconds)).multiply(new BigDecimal(1000000000)); + int nanos = bd.intValue(); + + // Convert to millis + long millis = seconds * 1000; + if (nanos < 0) { + millis -= 1000; + nanos += 1000000000; + } + Timestamp t = new Timestamp(millis); + + // Set remaining fractional portion to nanos + t.setNanos(nanos); + return t; + } catch (NumberFormatException nfe) { + return null; + } catch (IllegalArgumentException iae) { + return null; + } + } + + /** + * Take a HiveDecimal and return the timestamp representation where the fraction part is the + * nanoseconds and integer part is the number of seconds. + * @param dec + * @return + */ + public static Timestamp decimalToTimestamp(HiveDecimal dec) { + + HiveDecimalWritable nanosWritable = new HiveDecimalWritable(dec); + nanosWritable.mutateFractionPortion(); // Clip off seconds portion. + nanosWritable.mutateScaleByPowerOfTen(9); // Bring nanoseconds into integer portion. + if (!nanosWritable.isSet() || !nanosWritable.isInt()) { + return null; + } + int nanos = nanosWritable.intValue(); + if (nanos < 0) { + nanos += 1000000000; + } + nanosWritable.setFromLong(nanos); + + HiveDecimalWritable nanoInstant = new HiveDecimalWritable(dec); + nanoInstant.mutateScaleByPowerOfTen(9); + + nanoInstant.mutateSubtract(nanosWritable); + nanoInstant.mutateScaleByPowerOfTen(-9); // Back to seconds. + if (!nanoInstant.isSet() || !nanoInstant.isLong()) { + return null; + } + long seconds = nanoInstant.longValue(); + Timestamp t = new Timestamp(seconds * 1000); + t.setNanos(nanos); + return t; + } + + /** + * Take a HiveDecimalWritable and return the timestamp representation where the fraction part + * is the nanoseconds and integer part is the number of seconds. + * + * This is a HiveDecimalWritable variation with supplied scratch objects. + * @param decWritable + * @param scratchDecWritable1 + * @param scratchDecWritable2 + * @return + */ + public static Timestamp decimalToTimestamp( + HiveDecimalWritable decWritable, + HiveDecimalWritable scratchDecWritable1, HiveDecimalWritable scratchDecWritable2) { + + HiveDecimalWritable nanosWritable = scratchDecWritable1; + nanosWritable.set(decWritable); + nanosWritable.mutateFractionPortion(); // Clip off seconds portion. + nanosWritable.mutateScaleByPowerOfTen(9); // Bring nanoseconds into integer portion. + if (!nanosWritable.isSet() || !nanosWritable.isInt()) { + return null; + } + int nanos = nanosWritable.intValue(); + if (nanos < 0) { + nanos += 1000000000; + } + nanosWritable.setFromLong(nanos); + + HiveDecimalWritable nanoInstant = scratchDecWritable2; + nanoInstant.set(decWritable); + nanoInstant.mutateScaleByPowerOfTen(9); + + nanoInstant.mutateSubtract(nanosWritable); + nanoInstant.mutateScaleByPowerOfTen(-9); // Back to seconds. + if (!nanoInstant.isSet() || !nanoInstant.isLong()) { + return null; + } + long seconds = nanoInstant.longValue(); + + Timestamp timestamp = new Timestamp(seconds * 1000L); + timestamp.setNanos(nanos); + return timestamp; + } + + public static Timestamp decimalToTimestamp(HiveDecimalV1 dec) { + try { + BigDecimal nanoInstant = dec.bigDecimalValue().multiply(BILLION_BIG_DECIMAL); + int nanos = nanoInstant.remainder(BILLION_BIG_DECIMAL).intValue(); + if (nanos < 0) { + nanos += 1000000000; + } + long seconds = + nanoInstant.subtract(new BigDecimal(nanos)).divide(BILLION_BIG_DECIMAL).longValue(); + Timestamp t = new Timestamp(seconds * 1000); + t.setNanos(nanos); + + return t; + } catch (NumberFormatException nfe) { + return null; + } catch (IllegalArgumentException iae) { + return null; + } + } + + /** + * Rounds the number of milliseconds relative to the epoch down to the nearest whole number of + * seconds. 500 would round to 0, -500 would round to -1. + */ + public static long millisToSeconds(long millis) { + if (millis >= 0) { + return millis / 1000; + } else { + return (millis - 999) / 1000; + } + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java new file mode 100644 index 0000000000000000000000000000000000000000..637720a35e830b26b1c1440a40bb9f9edc9005fe --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java @@ -0,0 +1,225 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.io; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.sql.Date; +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.TimeZone; +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.io.WritableUtils; + + +/** + * DateWritable + * Writable equivalent of java.sql.Date. + * + * Dates are of the format + * YYYY-MM-DD + * + */ +public class DateWritable implements WritableComparable { + + private static final long MILLIS_PER_DAY = TimeUnit.DAYS.toMillis(1); + + // Local time zone. Store separately because Calendar would clone it. + // Java TimeZone has no mention of thread safety. Use thread local instance to be safe. + private static final ThreadLocal LOCAL_TIMEZONE = new ThreadLocal() { + @Override + protected TimeZone initialValue() { + return Calendar.getInstance().getTimeZone(); + } + }; + + private static final ThreadLocal UTC_CALENDAR = new ThreadLocal() { + @Override + protected Calendar initialValue() { + return new GregorianCalendar(TimeZone.getTimeZone("UTC")); + } + }; + private static final ThreadLocal LOCAL_CALENDAR = new ThreadLocal() { + @Override + protected Calendar initialValue() { + return Calendar.getInstance(); + } + }; + + // Internal representation is an integer representing day offset from our epoch value 1970-01-01 + private int daysSinceEpoch = 0; + + /* Constructors */ + public DateWritable() { + } + + public DateWritable(DateWritable d) { + set(d); + } + + public DateWritable(Date d) { + set(d); + } + + public DateWritable(int d) { + set(d); + } + + /** + * Set the DateWritable based on the days since epoch date. + * @param d integer value representing days since epoch date + */ + public void set(int d) { + daysSinceEpoch = d; + } + + /** + * Set the DateWritable based on the year/month/day of the date in the local timezone. + * @param d Date value + */ + public void set(Date d) { + if (d == null) { + daysSinceEpoch = 0; + return; + } + + set(dateToDays(d)); + } + + public void set(DateWritable d) { + set(d.daysSinceEpoch); + } + + /** + * @return Date value corresponding to the date in the local time zone + */ + public Date get() { + return get(true); + } + + // TODO: we should call this more often. In theory, for DATE type, time should never matter, but + // it's hard to tell w/some code paths like UDFs/OIs etc. that are used in many places. + public Date get(boolean doesTimeMatter) { + return new Date(daysToMillis(daysSinceEpoch, doesTimeMatter)); + } + + public int getDays() { + return daysSinceEpoch; + } + + /** + * + * @return time in seconds corresponding to this DateWritable + */ + public long getTimeInSeconds() { + return get().getTime() / 1000; + } + + public static Date timeToDate(long l) { + return new Date(l * 1000); + } + + public static long daysToMillis(int d) { + return daysToMillis(d, true); + } + + public static long daysToMillis(int d, boolean doesTimeMatter) { + // What we are trying to get is the equivalent of new Date(ymd).getTime() in the local tz, + // where ymd is whatever d represents. How it "works" is this. + // First we get the UTC midnight for that day (which always exists, a small island of sanity). + long utcMidnight = d * MILLIS_PER_DAY; + // Now we take a local TZ offset at midnight UTC. Say we are in -4; that means (surprise + // surprise) that at midnight UTC it was 20:00 in local. So far we are on firm ground. + long utcMidnightOffset = LOCAL_TIMEZONE.get().getOffset(utcMidnight); + // And now we wander straight into the swamp, when instead of adding, we subtract it from UTC + // midnight to supposedly get local midnight (in the above case, 4:00 UTC). Of course, given + // all the insane DST variations, where we actually end up is anyone's guess. + long hopefullyMidnight = utcMidnight - utcMidnightOffset; + // Then we determine the local TZ offset at that magical time. + long offsetAtHM = LOCAL_TIMEZONE.get().getOffset(hopefullyMidnight); + // If the offsets are the same, we assume our initial jump did not cross any DST boundaries, + // and is thus valid. Both times flowed at the same pace. We congratulate ourselves and bail. + if (utcMidnightOffset == offsetAtHM) return hopefullyMidnight; + // Alas, we crossed some DST boundary. If the time of day doesn't matter to the caller, we'll + // simply get the next day and go back half a day. This is not ideal but seems to work. + if (!doesTimeMatter) return daysToMillis(d + 1) - (MILLIS_PER_DAY >> 1); + // Now, we could get previous and next day, figure our how many hours were inserted or removed, + // and from which of the days, etc. But at this point our gun is pointing straight at our foot, + // so let's just go the safe, expensive way. + Calendar utc = UTC_CALENDAR.get(), local = LOCAL_CALENDAR.get(); + utc.setTimeInMillis(utcMidnight); + local.set(utc.get(Calendar.YEAR), utc.get(Calendar.MONTH), utc.get(Calendar.DAY_OF_MONTH)); + return local.getTimeInMillis(); + } + + public static int millisToDays(long millisLocal) { + // We assume millisLocal is midnight of some date. What we are basically trying to do + // here is go from local-midnight to UTC-midnight (or whatever time that happens to be). + long millisUtc = millisLocal + LOCAL_TIMEZONE.get().getOffset(millisLocal); + int days; + if (millisUtc >= 0L) { + days = (int) (millisUtc / MILLIS_PER_DAY); + } else { + days = (int) ((millisUtc - 86399999 /*(MILLIS_PER_DAY - 1)*/) / MILLIS_PER_DAY); + } + return days; + } + + public static int dateToDays(Date d) { + // convert to equivalent time in UTC, then get day offset + long millisLocal = d.getTime(); + return millisToDays(millisLocal); + } + + @Override + public void readFields(DataInput in) throws IOException { + daysSinceEpoch = WritableUtils.readVInt(in); + } + + @Override + public void write(DataOutput out) throws IOException { + WritableUtils.writeVInt(out, daysSinceEpoch); + } + + @Override + public int compareTo(DateWritable d) { + return daysSinceEpoch - d.daysSinceEpoch; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof DateWritable)) { + return false; + } + return compareTo((DateWritable) o) == 0; + } + + @Override + public String toString() { + // For toString, the time does not matter + return get(false).toString(); + } + + @Override + public int hashCode() { + return daysSinceEpoch; + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveBaseCharWritable.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveBaseCharWritable.java new file mode 100644 index 0000000000000000000000000000000000000000..57feaa54afec7b00539081932419f0df8751a72a --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveBaseCharWritable.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.io; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.Text; + +public abstract class HiveBaseCharWritable { + protected Text value = new Text(); + + public HiveBaseCharWritable() { + } + + public int getCharacterLength() { + return getTextUtfLength(value); + } + + /** + * Access to the internal Text member. Use with care. + * @return + */ + public Text getTextValue() { + return value; + } + + public void readFields(DataInput in) throws IOException { + value.readFields(in); + } + + public void write(DataOutput out) throws IOException { + value.write(out); + } + + public boolean equals(Object obj) { + if (obj == null || (obj.getClass() != this.getClass())) { + return false; + } + return value.equals(((HiveBaseCharWritable)obj).value); + } + + public int hashCode() { + return value.hashCode(); + } + + /** + * Checks if b is the first byte of a UTF-8 character. + * + */ + public static boolean isUtfStartByte(byte b) { + return (b & 0xC0) != 0x80; + } + + public static int getTextUtfLength(Text t) { + byte[] data = t.getBytes(); + int len = 0; + for (int i = 0; i < t.getLength(); i++) { + if (isUtfStartByte(data[i])) { + len++; + } + } + return len; + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveCharWritable.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveCharWritable.java new file mode 100644 index 0000000000000000000000000000000000000000..9af2336515932bc1da6ecb589ebbbdd1919e3ec4 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveCharWritable.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.io; + +import org.apache.hadoop.hive.common.type.HiveBaseChar; +import org.apache.hadoop.hive.common.type.HiveChar; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparable; + +/** + * HiveCharWritable. + * String values will be padded to full char length. + * Character legnth, comparison, hashCode should ignore trailing spaces. + */ +public class HiveCharWritable extends HiveBaseCharWritable + implements WritableComparable { + + public HiveCharWritable() { + } + + public HiveCharWritable(HiveChar hc) { + set(hc); + } + + public HiveCharWritable(HiveCharWritable hcw) { + set(hcw); + } + + public void set(HiveChar val) { + set(val.getValue(), -1); + } + + public void set(String val) { + set(val, -1); + } + + public void set(HiveCharWritable val) { + value.set(val.value); + } + + public void set(HiveCharWritable val, int maxLength) { + set(val.getHiveChar(), maxLength); + } + + public void set(HiveChar val, int len) { + set(val.getValue(), len); + } + + public void set(String val, int maxLength) { + value.set(HiveBaseChar.getPaddedValue(val, maxLength)); + } + + public HiveChar getHiveChar() { + // Copy string value as-is + return new HiveChar(value.toString(), -1); + } + + public void enforceMaxLength(int maxLength) { + if (getCharacterLength()!=maxLength) + set(getHiveChar(), maxLength); + } + + public Text getStrippedValue() { + // A lot of these methods could be done more efficiently by operating on the Text value + // directly, rather than converting to HiveChar. + return new Text(getHiveChar().getStrippedValue()); + } + + public Text getPaddedValue() { + return getTextValue(); + } + + public int getCharacterLength() { + return HiveBaseCharWritable.getTextUtfLength(getStrippedValue()); + } + + public int compareTo(HiveCharWritable rhs) { + return getStrippedValue().compareTo(rhs.getStrippedValue()); + } + + public boolean equals(Object rhs) { + if (rhs == this) { + return true; + } + if (rhs == null || rhs.getClass() != getClass()) { + return false; + } + return this.getStrippedValue().equals(((HiveCharWritable) rhs).getStrippedValue()); + } + + public int hashCode() { + return getStrippedValue().hashCode(); + } + + @Override + public String toString() { + return getPaddedValue().toString(); + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java new file mode 100644 index 0000000000000000000000000000000000000000..7a3bf4d08f389cd814565672cf454816c716e464 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java @@ -0,0 +1,1099 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.io; + +import java.util.Arrays; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.IOException; +import java.math.BigInteger; + +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.FastHiveDecimal; +import org.apache.hadoop.hive.common.type.FastHiveDecimalImpl; +import org.apache.hadoop.hive.common.type.HiveDecimalVersionV2; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.io.WritableUtils; + +/** + * A mutable decimal. + *

+ * ------------------------------------- Version 2 ------------------------------------------------ + *

+ * This is the 2nd major version of HiveDecimalWritable called V2. The previous version has been + * renamed to HiveDecimalWritableV1 and is kept as a test and behavior reference. + *

+ * For good performance we do not represent the decimal using a byte array containing BigInteger + * bytes like the previous version V1 did. Instead V2 HiveDecimalWritable is is a private subclass + * of the same fast decimal class also used by HiveDecimal. So it stores the decimal value + * directly. + *

+ * Many of the methods of HiveDecimal have been added to HiveDecimalWritable in V2 so code can + * modify the decimal instead of calling getHiveDecimal(), doing operations on HiveDecimal, and then + * setting HiveDecimalWritable back to the result. + *

+ * Operations that modify have a prefix of "mutate" in their name. For example mutateAdd is used + * instead of the immutable operation add in HiveDecimal that returns a new decimal object. + *

+ * This should have much better performance. + *

+ * The original V1 public methods and fields are annotated with @HiveDecimalWritableVersionV1; new + * public methods and fields are annotated with @HiveDecimalWritableVersionV2. + * + */ +public final class HiveDecimalWritable extends FastHiveDecimal + implements WritableComparable { + + // Is the decimal value currently valid? + private boolean isSet; + + /* + * Scratch arrays used in fastBigIntegerBytes calls for better performance. + */ + + // An optional long array of length FastHiveDecimal.FAST_SCRATCH_LONGS_LEN. + private long[] internalScratchLongs; + + // An optional byte array of FastHiveDecimal.FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES. + private byte[] internalScratchBuffer; + + /** + * Create a decimal writable with no current value (i.e. isSet() will return false). + */ + @HiveDecimalWritableVersionV1 + public HiveDecimalWritable() { + super(); + isSet = false; + internalScratchLongs = null; + internalScratchBuffer = null; + } + + /** + * Create a decimal writable with an initial value from a String. + *

+ * If the conversion from String to decimal is successful, isSet() will return true. + * + */ + @HiveDecimalWritableVersionV1 + public HiveDecimalWritable(String string) { + super(); + isSet = fastSetFromString(string, false); + if (!isSet) { + fastReset(); + } + } + + /** + * Create a decimal writable with an initial value from BigInteger bytes and a specified scale. + *

+ * If the conversion to decimal is successful, isSet() will return true. + * + */ + @HiveDecimalWritableVersionV1 + public HiveDecimalWritable(byte[] bigIntegerBytes, int scale) { + super(); + setFromBigIntegerBytesAndScale(bigIntegerBytes, scale); + } + + /** + * Create a decimal writable with an initial value from another decimal writable. + *

+ * If the supplied writable has a value, isSet() will return true. + * + */ + @HiveDecimalWritableVersionV1 + public HiveDecimalWritable(HiveDecimalWritable writable) { + super(); + set(writable); + } + + /** + * Create a decimal writable with an initial value from a HiveDecimal. + *

+ * Afterwards, the isSet() method will return true, unless value is null. + * + */ + @HiveDecimalWritableVersionV1 + public HiveDecimalWritable(HiveDecimal value) { + super(); + set(value); + } + + /** + * Create a decimal writable with an initial value from a long with scale 0. + *

+ * Afterwards, the isSet() method will return true. + * + */ + @HiveDecimalWritableVersionV1 + public HiveDecimalWritable(long longValue) { + super(); + setFromLong(longValue); + } + + /** + * Set the writable's current value to a HiveDecimal's value. + *

+ * Afterwards, the isSet() method will return true, unless value is null. + * + */ + @HiveDecimalWritableVersionV1 + public void set(HiveDecimal value) { + if (value == null) { + fastReset(); + isSet = false; + } else { + fastSet(value); + isSet = true; + } + } + + /** + * Set the writable's current value to a HiveDecimal's value with a specified precision / scale + * enforced. + *

+ * Afterwards, the isSet() method will return true, unless value is null or value didn't fit within + * maxPrecision / maxScale. + * + */ + @HiveDecimalWritableVersionV1 + public void set(HiveDecimal value, int maxPrecision, int maxScale) { + set(value); + if (isSet) { + isSet = fastEnforcePrecisionScale(maxPrecision, maxScale); + if (!isSet) { + fastReset(); + } + } + } + + /** + * Set the writable's current value to the value in a another decimal writable. + *

+ * If the supplied writable has a value, isSet() will return true. + * + */ + @HiveDecimalWritableVersionV1 + public void set(HiveDecimalWritable writable) { + if (writable == null || !writable.isSet()) { + fastReset(); + isSet = false; + } else { + fastSet(writable); + isSet = true; + } + } + + /** + * Set a decimal writable's value from BigInteger bytes and a specified scale. + *

+ * If the conversion to decimal is successful, isSet() will return true. + * + */ + @HiveDecimalWritableVersionV1 + public void set(byte[] bigIntegerBytes, int scale) { + setFromBigIntegerBytesAndScale(bigIntegerBytes, scale); + } + + /** + * Set the writable's current value to a writable's value with a specified precision / scale + * enforced. + *

+ * The isSet() method will return true, unless value is null or value didn't fit within + * maxPrecision / maxScale. + * + */ + @HiveDecimalWritableVersionV2 + public void set(HiveDecimalWritable writable, int maxPrecision, int maxScale) { + set(writable); + if (isSet) { + isSet = fastEnforcePrecisionScale(maxPrecision, maxScale); + if (!isSet) { + fastReset(); + } + } + } + + /** + * Set a decimal writable's value to a long's value with scale 0. + *

+ * Afterwards, the isSet() method will return true since all long values fit in a decimal. + * + */ + @HiveDecimalWritableVersionV2 + public void setFromLong(long longValue) { + fastReset(); + fastSetFromLong(longValue); + isSet = true; + } + + /** + * Set a decimal writable's value to a doubles value. + *

+ * Afterwards, the isSet() method will return true if the double to decimal conversion was successful. + * + */ + @HiveDecimalWritableVersionV2 + public void setFromDouble(double doubleValue) { + fastReset(); + isSet = fastSetFromDouble(doubleValue); + if (!isSet) { + fastReset(); + } + } + + /** + * Set the writable's current value to the decimal in a UTF-8 byte slice. + *

+ * Afterwards, the isSet() method will return true, unless byte slice could not be converted. + * + */ + @HiveDecimalWritableVersionV2 + public void setFromBytes(byte[] bytes, int offset, int length) { + fastReset(); + isSet = fastSetFromBytes(bytes, offset, length, false); + if (!isSet) { + fastReset(); + } + } + + @HiveDecimalWritableVersionV2 + public void setFromBytes(byte[] bytes, int offset, int length, boolean trimBlanks) { + fastReset(); + isSet = fastSetFromBytes(bytes, offset, length, trimBlanks); + if (!isSet) { + fastReset(); + } + } + + /** + * Set the writable's current value to the decimal digits only in a UTF-8 byte slice, a sign + * flag, and a scale. + *

+ * Afterwards, the isSet() method will return true, unless byte slice etc could not be converted. + * + */ + @HiveDecimalWritableVersionV2 + public void setFromDigitsOnlyBytesWithScale( + boolean isNegative, byte[] bytes, int offset, int length, int scale) { + fastReset(); + isSet = fastSetFromDigitsOnlyBytesAndScale(isNegative, bytes, offset, length, scale); + if (!isSet) { + fastReset(); + } + } + + /** + * Set the writable's current value to the signed value from BigInteger bytes and a specified + * scale. + *

+ * Afterwards, the isSet() method will return true, unless conversion failed. + * + */ + @HiveDecimalWritableVersionV2 + public void setFromBigIntegerBytesAndScale(byte[] bigIntegerBytes, int scale) { + fastReset(); + isSet = fastSetFromBigIntegerBytesAndScale(bigIntegerBytes, 0, bigIntegerBytes.length, scale); + if (!isSet) { + fastReset(); + } + } + + @HiveDecimalWritableVersionV2 + public void setFromBigIntegerBytesAndScale( + byte[] bigIntegerBytes, int offset, int length, int scale) { + fastReset(); + isSet = fastSetFromBigIntegerBytesAndScale(bigIntegerBytes, offset, length, scale); + if (!isSet) { + fastReset(); + } + } + + /** + * Set the writable's current value to the long's value at a specified + * scale. + *

+ * Afterwards, the isSet() method will return true, unless conversion failed. + * + */ + @HiveDecimalWritableVersionV2 + public void setFromLongAndScale(long longValue, int scale) { + fastReset(); + isSet = fastSetFromLongAndScale(longValue, scale); + if (!isSet) { + fastReset(); + } + } + + /** + * Does this writable have a current value? + *

+ * A return of false means a current value wasn't set, or an operation like mutateAdd overflowed, + * or a set* method couldn't convert the input value, etc. + * + */ + @HiveDecimalWritableVersionV2 + public boolean isSet() { + return isSet; + } + + /** + * Returns a HiveDecimal for the writable's current value. + *

+ * Returns null if the writable isn't set. + * + */ + @HiveDecimalWritableVersionV1 + public HiveDecimal getHiveDecimal() { + if (!isSet) { + return null; + } + HiveDecimal result = HiveDecimal.createFromFast(this); + return result; + } + + /** + * Get a HiveDecimal instance from the writable and constraint it with maximum precision/scale. + *

+ * @param maxPrecision maximum precision + * @param maxScale maximum scale + * @return HiveDecimal instance + */ + @HiveDecimalWritableVersionV1 + public HiveDecimal getHiveDecimal(int maxPrecision, int maxScale) { + if (!isSet) { + return null; + } + HiveDecimal dec = HiveDecimal.createFromFast(this); + HiveDecimal result = HiveDecimal.enforcePrecisionScale(dec, maxPrecision, maxScale); + return result; + } + + /** + * Standard Writable method that deserialize the fields of this object from a DataInput. + * + */ + @HiveDecimalWritableVersionV1 + @Override + public void readFields(DataInput in) throws IOException { + int scale = WritableUtils.readVInt(in); + int byteArrayLen = WritableUtils.readVInt(in); + byte[] bytes = new byte[byteArrayLen]; + in.readFully(bytes); + + fastReset(); + if (!fastSetFromBigIntegerBytesAndScale(bytes, 0, bytes.length, scale)) { + throw new IOException("Couldn't convert decimal"); + } + isSet = true; + } + + /** + * Standard Writable method that serialize the fields of this object to a DataOutput. + * + */ + @HiveDecimalWritableVersionV1 + @Override + public void write(DataOutput out) throws IOException { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + + if (internalScratchLongs == null) { + internalScratchLongs = new long[FastHiveDecimal.FAST_SCRATCH_LONGS_LEN]; + internalScratchBuffer = new byte[FastHiveDecimal.FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; + } + + write(out, internalScratchLongs, internalScratchBuffer); + } + + + /** + * A variation of the standard Writable method that serialize the fields of this object to a + * DataOutput with scratch buffers for good performance. + *

+ * Allocate scratchLongs with HiveDecimal.SCRATCH_LONGS_LEN longs. + * And, allocate scratch buffer with HiveDecimal.SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes. + * + */ + @HiveDecimalWritableVersionV2 + public void write( + DataOutput out, + long[] scratchLongs, byte[] scratchBuffer) throws IOException { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + + WritableUtils.writeVInt(out, fastScale()); + + int byteLength = + fastBigIntegerBytes( + scratchLongs, scratchBuffer); + if (byteLength == 0) { + throw new RuntimeException("Couldn't convert decimal to binary"); + } + + WritableUtils.writeVInt(out, byteLength); + out.write(scratchBuffer, 0, byteLength); + } + + /** + * See the comments for HiveDecimal.serializationUtilsRead. + */ + @HiveDecimalWritableVersionV2 + public boolean serializationUtilsRead( + InputStream inputStream, int scale, + byte[] scratchBytes) + throws IOException { + fastReset(); + isSet = + fastSerializationUtilsRead( + inputStream, + scale, + scratchBytes); + return isSet; + } + + /** + * See the comments for HiveDecimal.serializationUtilsWrite. + */ + @HiveDecimalWritableVersionV2 + public boolean serializationUtilsWrite( + OutputStream outputStream, + long[] scratchLongs) + throws IOException { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return + fastSerializationUtilsWrite( + outputStream, + scratchLongs); + } + + /* + * Maximum number of decimal digits in a decimal64 long. + */ + @HiveDecimalWritableVersionV2 + public static final int DECIMAL64_DECIMAL_DIGITS = FastHiveDecimalImpl.DECIMAL64_DECIMAL_DIGITS; + + /* + * Test whether a precision will fit within a decimal64 (64-bit signed long with <= 18 decimal + * digits). + */ + @HiveDecimalWritableVersionV2 + public static boolean isPrecisionDecimal64(int precision) { + return (precision <= DECIMAL64_DECIMAL_DIGITS); + } + + /* + * Return the maximum absolute decimal64 value for a precision. + */ + @HiveDecimalWritableVersionV2 + public static long getDecimal64AbsMax(int precision) { + return FastHiveDecimalImpl.getDecimal64AbsMax(precision); + } + + /* + * Deserializes 64-bit decimals up to the maximum 64-bit precision (18 decimal digits). + * + * NOTE: Major assumption: the input decimal64 has already been bounds checked and a least + * has a precision <= DECIMAL64_DECIMAL_DIGITS. We do not bounds check here for better + * performance. You can bounds check beforehand with: + * Math.abs(decimal64Long) <= getDecimal64AbsMax(precision) + */ + @HiveDecimalWritableVersionV2 + public void deserialize64( + long decimal64Long, int scale) { + fastDeserialize64(decimal64Long, scale); + isSet = true; + } + + /* + * Serializes decimal64 up to the maximum 64-bit precision (18 decimal digits). + * + * NOTE: Major assumption: the fast decimal has already been bounds checked and a least + * has a precision <= DECIMAL64_DECIMAL_DIGITS. We do not bounds check here for better + * performance. + */ + @HiveDecimalWritableVersionV2 + public long serialize64(int scale) { + return fastSerialize64(scale); + } + + @HiveDecimalWritableVersionV2 + public boolean isValid() { + if (!isSet) { + return false; + } + return FastHiveDecimalImpl.fastIsValid(this); + } + + /** + * Returns the length of the decimal converted to bytes. + * Call bigIntegerBytesBuffer() to get a reference to the converted bytes. + * + */ + @HiveDecimalWritableVersionV2 + public int bigIntegerBytesInternalScratch() { + + if (!isSet()) { + throw new RuntimeException("no value set"); + } + + if (internalScratchLongs == null) { + internalScratchLongs = new long[FastHiveDecimal.FAST_SCRATCH_LONGS_LEN]; + internalScratchBuffer = new byte[FastHiveDecimal.FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; + } + + int byteLength = + fastBigIntegerBytes( + internalScratchLongs, internalScratchBuffer); + if (byteLength == 0) { + throw new RuntimeException("Couldn't convert decimal to binary"); + } + return byteLength; + } + + /** + * Returns the scratch array containing the result after a call to bigIntegerBytesInternalScratch. + * + */ + @HiveDecimalWritableVersionV2 + public byte[] bigIntegerBytesInternalScratchBuffer() { + return internalScratchBuffer; + } + + /** + * Allocate scratchLongs with HiveDecimal.SCRATCH_LONGS_LEN longs. + * And, allocate scratch scratchBuffer with HiveDecimal.SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes. + * + */ + @HiveDecimalWritableVersionV2 + public byte[] bigIntegerBytesCopy( + long[] scratchLongs, byte[] scratchBuffer) { + + if (!isSet()) { + throw new RuntimeException("no value set"); + } + + int byteLength = + fastBigIntegerBytes( + scratchLongs, scratchBuffer); + if (byteLength == 0) { + throw new RuntimeException("Couldn't convert decimal to binary"); + } + return Arrays.copyOf(scratchBuffer, byteLength); + } + + @HiveDecimalWritableVersionV2 + public int bigIntegerBytes( + long[] scratchLongs, byte[] scratchBuffer) { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + int byteLength = + fastBigIntegerBytes( + scratchLongs, scratchBuffer); + return byteLength; + } + + @HiveDecimalWritableVersionV2 + public int signum() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastSignum(); + } + + @HiveDecimalWritableVersionV2 + public int precision() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastSqlPrecision(); + } + + @HiveDecimalWritableVersionV2 + public int rawPrecision() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastRawPrecision(); + } + + @HiveDecimalWritableVersionV2 + public int scale() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastScale(); + } + + @HiveDecimalWritableVersionV2 + public boolean isByte() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastIsByte(); + } + + @HiveDecimalWritableVersionV2 + public byte byteValue() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastByteValueClip(); + } + + @HiveDecimalWritableVersionV2 + public boolean isShort() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastIsShort(); + } + + @HiveDecimalWritableVersionV2 + public short shortValue() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastShortValueClip(); + } + + @HiveDecimalWritableVersionV2 + public boolean isInt() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastIsInt(); + } + + @HiveDecimalWritableVersionV2 + public int intValue() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastIntValueClip(); + } + + @HiveDecimalWritableVersionV2 + public boolean isLong() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastIsLong(); + } + + @HiveDecimalWritableVersionV2 + public long longValue() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastLongValueClip(); + } + + @HiveDecimalWritableVersionV2 + public float floatValue() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastFloatValue(); + } + + @HiveDecimalWritableVersionV2 + public double doubleValue() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastDoubleValue(); + } + + //----------------------------------------------------------------------------------------------- + // Mutate operations. + //----------------------------------------------------------------------------------------------- + + @HiveDecimalWritableVersionV2 + public void mutateAbs() { + if (!isSet) { + return; + } + fastAbs(); + } + + @HiveDecimalWritableVersionV2 + public void mutateNegate() { + if (!isSet) { + return; + } + fastNegate(); + } + + @HiveDecimalWritableVersionV2 + public void mutateAdd(HiveDecimalWritable decWritable) { + if (!isSet || !decWritable.isSet) { + isSet = false; + return; + } + isSet = + fastAdd(decWritable, this); + } + + @HiveDecimalWritableVersionV2 + public void mutateAdd(HiveDecimal dec) { + if (dec == null) { + + // Can't add NULL. + isSet = false; + return; + } + if (!isSet) { + return; + } + isSet = + fastAdd(dec, this); + } + + @HiveDecimalWritableVersionV2 + public void mutateSubtract(HiveDecimalWritable decWritable) { + if (!isSet || !decWritable.isSet) { + isSet = false; + return; + } + isSet = + fastSubtract(decWritable, this); + } + + @HiveDecimalWritableVersionV2 + public void mutateSubtract(HiveDecimal dec) { + if (dec == null) { + + // Can't subtract NULL. + isSet = false; + return; + } + if (!isSet) { + return; + } + isSet = + fastSubtract(dec, this); + } + + @HiveDecimalWritableVersionV2 + public void mutateMultiply(HiveDecimalWritable decWritable) { + if (!isSet || !decWritable.isSet) { + isSet = false; + return; + } + isSet = + fastMultiply(decWritable, this); + } + + @HiveDecimalWritableVersionV2 + public void mutateMultiply(HiveDecimal dec) { + if (dec == null) { + + // Can't multiply NULL. + isSet = false; + return; + } + if (!isSet) { + return; + } + isSet = + fastMultiply(dec, this); + } + + @HiveDecimalWritableVersionV2 + public void mutateDivide(HiveDecimalWritable decWritable) { + if (!isSet || !decWritable.isSet) { + isSet = false; + return; + } + isSet = + fastDivide(decWritable, this); + } + + @HiveDecimalWritableVersionV2 + public void mutateDivide(HiveDecimal dec) { + if (dec == null) { + + // Can't divide NULL. + isSet = false; + return; + } + if (!isSet) { + return; + } + isSet = + fastDivide(dec, this); + + } + + @HiveDecimalWritableVersionV2 + public void mutateRemainder(HiveDecimalWritable decWritable) { + if (!isSet || !decWritable.isSet) { + isSet = false; + return; + } + isSet = + fastRemainder(decWritable, this); + } + + @HiveDecimalWritableVersionV2 + public void mutateRemainder(HiveDecimal dec) { + if (dec == null) { + + // Can't do remainder on NULL. + isSet = false; + return; + } + if (!isSet) { + return; + } + isSet = + fastRemainder(dec, this); + } + + @HiveDecimalWritableVersionV2 + public void mutateScaleByPowerOfTen(int power) { + if (!isSet) { + return; + } + isSet = fastScaleByPowerOfTen(power, this); + } + + @HiveDecimalWritableVersionV2 + public void mutateFractionPortion() { + if (!isSet) { + return; + } + fastFractionPortion(); + } + + @HiveDecimalWritableVersionV2 + public void mutateIntegerPortion() { + if (!isSet) { + return; + } + fastIntegerPortion(); + } + + //----------------------------------------------------------------------------------------------- + // Standard overrides methods. + //----------------------------------------------------------------------------------------------- + + @HiveDecimalWritableVersionV1 + @Override + public int compareTo(HiveDecimalWritable writable) { + if (!isSet() || writable == null || !writable.isSet()) { + throw new RuntimeException("Invalid comparision operand(s)"); + } + return fastCompareTo(writable); + } + + @HiveDecimalWritableVersionV2 + public int compareTo(HiveDecimal dec) { + if (!isSet() || dec == null) { + throw new RuntimeException("Invalid comparision operand(s)"); + } + return fastCompareTo(dec); + } + + @HiveDecimalWritableVersionV2 + public static int compareTo(HiveDecimal dec, HiveDecimalWritable writable) { + if (dec == null || !writable.isSet()) { + throw new RuntimeException("Invalid comparision operand(s)"); + } + return FastHiveDecimal.fastCompareTo(dec, writable); + } + + @HiveDecimalWritableVersionV2 + public int toBytes(byte[] scratchBuffer) { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastToBytes(scratchBuffer); + } + + @HiveDecimalWritableVersionV1 + @Override + public String toString() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastToString(); + } + + @HiveDecimalWritableVersionV2 + public String toString( + byte[] scratchBuffer) { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + if (fastSerializationScale() != -1) { + + // Use the serialization scale and format the string with trailing zeroes (or + // round the decimal) if necessary. + return + fastToFormatString( + fastSerializationScale(), + scratchBuffer); + } else { + return + fastToString(scratchBuffer); + } + } + + @HiveDecimalWritableVersionV2 + public String toFormatString( + int formatScale) { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return + fastToFormatString( + formatScale); + } + + @HiveDecimalWritableVersionV2 + public int toFormatBytes( + int formatScale, + byte[] scratchBuffer) { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return + fastToFormatBytes( + formatScale, + scratchBuffer); + } + + @HiveDecimalWritableVersionV2 + public int toDigitsOnlyBytes( + byte[] scratchBuffer) { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return + fastToDigitsOnlyBytes( + scratchBuffer); + } + + @HiveDecimalWritableVersionV1 + @Override + public boolean equals(Object other) { + if (!isSet) { + return false; + } + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + HiveDecimalWritable otherHiveDecWritable = (HiveDecimalWritable) other; + if (!otherHiveDecWritable.isSet()) { + return false; + } + return fastEquals((FastHiveDecimal) otherHiveDecWritable); + + } + + @HiveDecimalWritableVersionV2 + public int newFasterHashCode() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastNewFasterHashCode(); + } + + @HiveDecimalWritableVersionV1 + @Override + public int hashCode() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastHashCode(); + } + + private static final byte[] EMPTY_ARRAY = new byte[0]; + + @HiveDecimalWritableVersionV1 + public byte[] getInternalStorage() { + if (!isSet()) { + // don't break old callers that are trying to reuse storages + return EMPTY_ARRAY; + } + + if (internalScratchLongs == null) { + internalScratchLongs = new long[FastHiveDecimal.FAST_SCRATCH_LONGS_LEN]; + internalScratchBuffer = new byte[FastHiveDecimal.FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; + } + + return bigIntegerBytesCopy( + internalScratchLongs, internalScratchBuffer); + } + + @HiveDecimalWritableVersionV1 + public int getScale() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastScale(); + } + + @HiveDecimalWritableVersionV2 + public void mutateSetScale(int roundingPoint, int roundingMode) { + if (!isSet) { + return; + } + isSet = fastRound(roundingPoint, roundingMode, this); + if (!isSet) { + fastReset(); + } + } + + @HiveDecimalWritableVersionV2 + public boolean mutateEnforcePrecisionScale(int precision, int scale) { + if (!isSet) { + return false; + } + isSet = fastEnforcePrecisionScale(precision, scale); + if (!isSet) { + fastReset(); + } + return isSet; + } + + @HiveDecimalWritableVersionV1 + public static HiveDecimalWritable enforcePrecisionScale(HiveDecimalWritable writable, int precision, int scale) { + if (!writable.isSet) { + return null; + } + HiveDecimalWritable result = new HiveDecimalWritable(writable); + result.mutateEnforcePrecisionScale(precision, scale); + if (!result.isSet()) { + return null; + } + return result; + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableV1.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableV1.java new file mode 100644 index 0000000000000000000000000000000000000000..ec976e9022fdd07867159afbf115ae9b24ca1c35 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableV1.java @@ -0,0 +1,191 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.io; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.math.BigInteger; + +import org.apache.hadoop.hive.common.type.HiveDecimalV1; + +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.io.WritableUtils; + +public class HiveDecimalWritableV1 implements WritableComparable { + + private byte[] internalStorage = new byte[0]; + private int scale; + + @HiveDecimalWritableVersionV1 + public HiveDecimalWritableV1() { + } + + @HiveDecimalWritableVersionV1 + public HiveDecimalWritableV1(String value) { + set(HiveDecimalV1.create(value)); + } + + @HiveDecimalWritableVersionV1 + public HiveDecimalWritableV1(byte[] bytes, int scale) { + set(bytes, scale); + } + + @HiveDecimalWritableVersionV1 + public HiveDecimalWritableV1(HiveDecimalWritableV1 writable) { + set(writable.getHiveDecimal()); + } + + @HiveDecimalWritableVersionV1 + public HiveDecimalWritableV1(HiveDecimalV1 value) { + set(value); + } + + @HiveDecimalWritableVersionV1 + public HiveDecimalWritableV1(long value) { + set((HiveDecimalV1.create(value))); + } + + @HiveDecimalWritableVersionV1 + public void set(HiveDecimalV1 value) { + set(value.unscaledValue().toByteArray(), value.scale()); + } + + @HiveDecimalWritableVersionV1 + public void set(HiveDecimalV1 value, int maxPrecision, int maxScale) { + set(HiveDecimalV1.enforcePrecisionScale(value, maxPrecision, maxScale)); + } + + @HiveDecimalWritableVersionV1 + public void set(HiveDecimalWritableV1 writable) { + set(writable.getHiveDecimal()); + } + + @HiveDecimalWritableVersionV1 + public void set(byte[] bytes, int scale) { + this.internalStorage = bytes; + this.scale = scale; + } + + @HiveDecimalWritableVersionV1 + public HiveDecimalV1 getHiveDecimal() { + return HiveDecimalV1.create(new BigInteger(internalStorage), scale); + } + + /** + * Get a OldHiveDecimal instance from the writable and constraint it with maximum precision/scale. + * + * @param maxPrecision maximum precision + * @param maxScale maximum scale + * @return OldHiveDecimal instance + */ + @HiveDecimalWritableVersionV1 + public HiveDecimalV1 getHiveDecimal(int maxPrecision, int maxScale) { + return HiveDecimalV1.enforcePrecisionScale(HiveDecimalV1. + create(new BigInteger(internalStorage), scale), + maxPrecision, maxScale); + } + + @HiveDecimalWritableVersionV1 + @Override + public void readFields(DataInput in) throws IOException { + scale = WritableUtils.readVInt(in); + int byteArrayLen = WritableUtils.readVInt(in); + if (internalStorage.length != byteArrayLen) { + internalStorage = new byte[byteArrayLen]; + } + in.readFully(internalStorage); + } + + @HiveDecimalWritableVersionV1 + @Override + public void write(DataOutput out) throws IOException { + WritableUtils.writeVInt(out, scale); + WritableUtils.writeVInt(out, internalStorage.length); + out.write(internalStorage); + } + + @HiveDecimalWritableVersionV1 + @Override + public int compareTo(HiveDecimalWritableV1 that) { + return getHiveDecimal().compareTo(that.getHiveDecimal()); + } + + @HiveDecimalWritableVersionV1 + @Override + public String toString() { + return getHiveDecimal().toString(); + } + + @HiveDecimalWritableVersionV1 + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + HiveDecimalWritableV1 bdw = (HiveDecimalWritableV1) other; + + // 'equals' and 'compareTo' are not compatible with HiveDecimals. We want + // compareTo which returns true iff the numbers are equal (e.g.: 3.14 is + // the same as 3.140). 'Equals' returns true iff equal and the same scale + // is set in the decimals (e.g.: 3.14 is not the same as 3.140) + return getHiveDecimal().compareTo(bdw.getHiveDecimal()) == 0; + } + + @HiveDecimalWritableVersionV1 + @Override + public int hashCode() { + return getHiveDecimal().hashCode(); + } + + /* (non-Javadoc) + * In order to update a Decimal128 fast (w/o allocation) we need to expose access to the + * internal storage bytes and scale. + * @return + */ + @HiveDecimalWritableVersionV1 + public byte[] getInternalStorage() { + return internalStorage; + } + + /* (non-Javadoc) + * In order to update a Decimal128 fast (w/o allocation) we need to expose access to the + * internal storage bytes and scale. + */ + @HiveDecimalWritableVersionV1 + public int getScale() { + return scale; + } + + @HiveDecimalWritableVersionV1 + public static + HiveDecimalWritableV1 enforcePrecisionScale(HiveDecimalWritableV1 writable, + int precision, int scale) { + if (writable == null) { + return null; + } + + HiveDecimalV1 dec = + HiveDecimalV1.enforcePrecisionScale(writable.getHiveDecimal(), precision, + scale); + return dec == null ? null : new HiveDecimalWritableV1(dec); + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableVersionV1.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableVersionV1.java new file mode 100644 index 0000000000000000000000000000000000000000..1aaa6f074dfc4c704fe889bf76f486f916960422 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableVersionV1.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.io; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Marks methods including static methods and fields as being part of version 1 HiveDecimalWritable. + * + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +public @interface HiveDecimalWritableVersionV1 { + +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableVersionV2.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableVersionV2.java new file mode 100644 index 0000000000000000000000000000000000000000..3c2b6f936033fc9acd9ab499c7dc50742e3d023e --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableVersionV2.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.io; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Marks methods including static methods and fields as being part of version 2 HiveDecimalWritable. + * + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +public @interface HiveDecimalWritableVersionV2 { + +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java new file mode 100644 index 0000000000000000000000000000000000000000..270d97fa6a856916f95006f0a29506dc9391d9aa --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.io; + +import org.apache.hadoop.hive.common.type.HiveBaseChar; +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.io.WritableComparable; + +public class HiveVarcharWritable extends HiveBaseCharWritable + implements WritableComparable{ + + public HiveVarcharWritable() { + } + + public HiveVarcharWritable(HiveVarchar hc) { + set(hc); + } + + public HiveVarcharWritable(HiveVarcharWritable hcw) { + set(hcw); + } + + public void set(HiveVarchar val) { + set(val.getValue()); + } + + public void set(String val) { + set(val, -1); // copy entire string value + } + + public void set(HiveVarcharWritable val) { + value.set(val.value); + } + + public void set(HiveVarcharWritable val, int maxLength) { + set(val.getHiveVarchar(), maxLength); + } + + public void set(HiveVarchar val, int len) { + set(val.getValue(), len); + } + + public void set(String val, int maxLength) { + value.set(HiveBaseChar.enforceMaxLength(val, maxLength)); + } + + public HiveVarchar getHiveVarchar() { + return new HiveVarchar(value.toString(), -1); + } + + public void enforceMaxLength(int maxLength) { + if (getCharacterLength() > maxLength) { + set(value.toString(), maxLength); + } + } + + @Override + public int compareTo(HiveVarcharWritable rhs) { + return value.compareTo(rhs.value); + } + + @Override + public String toString() { + return value.toString(); + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hive/common/util/BloomFilter.java b/storage-api/hive-storage-api/src/java/org/apache/hive/common/util/BloomFilter.java new file mode 100644 index 0000000000000000000000000000000000000000..706b83409b2e8920af5ff6a6cc5d8f61398ff745 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hive/common/util/BloomFilter.java @@ -0,0 +1,395 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.common.util; + +import java.io.*; +import java.util.Arrays; + +/** + * BloomFilter is a probabilistic data structure for set membership check. BloomFilters are + * highly space efficient when compared to using a HashSet. Because of the probabilistic nature of + * bloom filter false positive (element not present in bloom filter but test() says true) are + * possible but false negatives are not possible (if element is present then test() will never + * say false). The false positive probability is configurable (default: 5%) depending on which + * storage requirement may increase or decrease. Lower the false positive probability greater + * is the space requirement. + * Bloom filters are sensitive to number of elements that will be inserted in the bloom filter. + * During the creation of bloom filter expected number of entries must be specified. If the number + * of insertions exceed the specified initial number of entries then false positive probability will + * increase accordingly. + * + * Internally, this implementation of bloom filter uses Murmur3 fast non-cryptographic hash + * algorithm. Although Murmur2 is slightly faster than Murmur3 in Java, it suffers from hash + * collisions for specific sequence of repeating bytes. Check the following link for more info + * https://code.google.com/p/smhasher/wiki/MurmurHash2Flaw + */ +public class BloomFilter { + public static final double DEFAULT_FPP = 0.05; + protected BitSet bitSet; + protected int numBits; + protected int numHashFunctions; + + public BloomFilter() { + } + + public BloomFilter(long expectedEntries) { + this(expectedEntries, DEFAULT_FPP); + } + + static void checkArgument(boolean expression, String message) { + if (!expression) { + throw new IllegalArgumentException(message); + } + } + + public BloomFilter(long expectedEntries, double fpp) { + checkArgument(expectedEntries > 0, "expectedEntries should be > 0"); + checkArgument(fpp > 0.0 && fpp < 1.0, "False positive probability should be > 0.0 & < 1.0"); + int nb = optimalNumOfBits(expectedEntries, fpp); + // make 'm' multiple of 64 + this.numBits = nb + (Long.SIZE - (nb % Long.SIZE)); + this.numHashFunctions = optimalNumOfHashFunctions(expectedEntries, numBits); + this.bitSet = new BitSet(numBits); + } + + /** + * A constructor to support rebuilding the BloomFilter from a serialized representation. + * @param bits - bits are used as such for bitset and are NOT copied, any changes to bits will affect bloom filter + * @param numFuncs - number of hash functions + */ + public BloomFilter(long[] bits, int numFuncs) { + super(); + // input long[] is set as such without copying, so any modification to the source will affect bloom filter + this.bitSet = new BitSet(bits); + this.numBits = bits.length * Long.SIZE; + this.numHashFunctions = numFuncs; + } + + static int optimalNumOfHashFunctions(long n, long m) { + return Math.max(1, (int) Math.round((double) m / n * Math.log(2))); + } + + static int optimalNumOfBits(long n, double p) { + return (int) (-n * Math.log(p) / (Math.log(2) * Math.log(2))); + } + + public void add(byte[] val) { + if (val == null) { + addBytes(val, -1, -1); + } else { + addBytes(val, 0, val.length); + } + } + + public void addBytes(byte[] val, int offset, int length) { + // We use the trick mentioned in "Less Hashing, Same Performance: Building a Better Bloom Filter" + // by Kirsch et.al. From abstract 'only two hash functions are necessary to effectively + // implement a Bloom filter without any loss in the asymptotic false positive probability' + + // Lets split up 64-bit hashcode into two 32-bit hash codes and employ the technique mentioned + // in the above paper + long hash64 = val == null ? Murmur3.NULL_HASHCODE : + Murmur3.hash64(val, offset, length); + addHash(hash64); + } + + private void addHash(long hash64) { + int hash1 = (int) hash64; + int hash2 = (int) (hash64 >>> 32); + + for (int i = 1; i <= numHashFunctions; i++) { + int combinedHash = hash1 + ((i + 1) * hash2); + // hashcode should be positive, flip all the bits if it's negative + if (combinedHash < 0) { + combinedHash = ~combinedHash; + } + int pos = combinedHash % numBits; + bitSet.set(pos); + } + } + + public void addString(String val) { + if (val == null) { + add(null); + } else { + add(val.getBytes()); + } + } + + public void addLong(long val) { + addHash(getLongHash(val)); + } + + public void addDouble(double val) { + addLong(Double.doubleToLongBits(val)); + } + + public boolean test(byte[] val) { + if (val == null) { + return testBytes(val, -1, -1); + } + return testBytes(val, 0, val.length); + } + + public boolean testBytes(byte[] val, int offset, int length) { + long hash64 = val == null ? Murmur3.NULL_HASHCODE : + Murmur3.hash64(val, offset, length); + return testHash(hash64); + } + + private boolean testHash(long hash64) { + int hash1 = (int) hash64; + int hash2 = (int) (hash64 >>> 32); + + for (int i = 1; i <= numHashFunctions; i++) { + int combinedHash = hash1 + ((i + 1) * hash2); + // hashcode should be positive, flip all the bits if it's negative + if (combinedHash < 0) { + combinedHash = ~combinedHash; + } + int pos = combinedHash % numBits; + if (!bitSet.get(pos)) { + return false; + } + } + return true; + } + + public boolean testString(String val) { + if (val == null) { + return test(null); + } else { + return test(val.getBytes()); + } + } + + public boolean testLong(long val) { + return testHash(getLongHash(val)); + } + + // Thomas Wang's integer hash function + // http://web.archive.org/web/20071223173210/http://www.concentric.net/~Ttwang/tech/inthash.htm + private long getLongHash(long key) { + key = (~key) + (key << 21); // key = (key << 21) - key - 1; + key = key ^ (key >> 24); + key = (key + (key << 3)) + (key << 8); // key * 265 + key = key ^ (key >> 14); + key = (key + (key << 2)) + (key << 4); // key * 21 + key = key ^ (key >> 28); + key = key + (key << 31); + return key; + } + + public boolean testDouble(double val) { + return testLong(Double.doubleToLongBits(val)); + } + + public long sizeInBytes() { + return getBitSize() / 8; + } + + public int getBitSize() { + return bitSet.getData().length * Long.SIZE; + } + + public int getNumHashFunctions() { + return numHashFunctions; + } + + public long[] getBitSet() { + return bitSet.getData(); + } + + @Override + public String toString() { + return "m: " + numBits + " k: " + numHashFunctions; + } + + /** + * Merge the specified bloom filter with current bloom filter. + * + * @param that - bloom filter to merge + */ + public void merge(BloomFilter that) { + if (this != that && this.numBits == that.numBits && this.numHashFunctions == that.numHashFunctions) { + this.bitSet.putAll(that.bitSet); + } else { + throw new IllegalArgumentException("BloomFilters are not compatible for merging." + + " this - " + this.toString() + " that - " + that.toString()); + } + } + + public void reset() { + this.bitSet.clear(); + } + + /** + * Serialize a bloom filter + * @param out output stream to write to + * @param bloomFilter BloomFilter that needs to be seralized + */ + public static void serialize(OutputStream out, BloomFilter bloomFilter) throws IOException { + /** + * Serialized BloomFilter format: + * 1 byte for the number of hash functions. + * 1 big endian int(That is how OutputStream works) for the number of longs in the bitset + * big endian longs in the BloomFilter bitset + */ + DataOutputStream dataOutputStream = new DataOutputStream(out); + dataOutputStream.writeByte(bloomFilter.numHashFunctions); + dataOutputStream.writeInt(bloomFilter.getBitSet().length); + for (long value : bloomFilter.getBitSet()) { + dataOutputStream.writeLong(value); + } + } + + /** + * Deserialize a bloom filter + * Read a byte stream, which was written by {@linkplain #serialize(OutputStream, BloomFilter)} + * into a {@code BloomFilter} + * @param in input bytestream + * @return deserialized BloomFilter + */ + public static BloomFilter deserialize(InputStream in) throws IOException { + if (in == null) { + throw new IOException("Input stream is null"); + } + + try { + DataInputStream dataInputStream = new DataInputStream(in); + int numHashFunc = dataInputStream.readByte(); + int numLongs = dataInputStream.readInt(); + long[] data = new long[numLongs]; + for (int i = 0; i < numLongs; i++) { + data[i] = dataInputStream.readLong(); + } + return new BloomFilter(data, numHashFunc); + } catch (RuntimeException e) { + IOException io = new IOException( "Unable to deserialize BloomFilter"); + io.initCause(e); + throw io; + } + } + + // Given a byte array consisting of a serialized BloomFilter, gives the offset (from 0) + // for the start of the serialized long values that make up the bitset. + // NumHashFunctions (1 byte) + NumBits (4 bytes) + public static final int START_OF_SERIALIZED_LONGS = 5; + + /** + * Merges BloomFilter bf2 into bf1. + * Assumes 2 BloomFilters with the same size/hash functions are serialized to byte arrays + * @param bf1Bytes + * @param bf1Start + * @param bf1Length + * @param bf2Bytes + * @param bf2Start + * @param bf2Length + */ + public static void mergeBloomFilterBytes( + byte[] bf1Bytes, int bf1Start, int bf1Length, + byte[] bf2Bytes, int bf2Start, int bf2Length) { + if (bf1Length != bf2Length) { + throw new IllegalArgumentException("bf1Length " + bf1Length + " does not match bf2Length " + bf2Length); + } + + // Validation on the bitset size/3 hash functions. + for (int idx = 0; idx < START_OF_SERIALIZED_LONGS; ++idx) { + if (bf1Bytes[bf1Start + idx] != bf2Bytes[bf2Start + idx]) { + throw new IllegalArgumentException("bf1 NumHashFunctions/NumBits does not match bf2"); + } + } + + // Just bitwise-OR the bits together - size/# functions should be the same, + // rest of the data is serialized long values for the bitset which are supposed to be bitwise-ORed. + for (int idx = START_OF_SERIALIZED_LONGS; idx < bf1Length; ++idx) { + bf1Bytes[bf1Start + idx] |= bf2Bytes[bf2Start + idx]; + } + } + + /** + * Bare metal bit set implementation. For performance reasons, this implementation does not check + * for index bounds nor expand the bit set size if the specified index is greater than the size. + */ + public class BitSet { + private final long[] data; + + public BitSet(long bits) { + this(new long[(int) Math.ceil((double) bits / (double) Long.SIZE)]); + } + + /** + * Deserialize long array as bit set. + * + * @param data - bit array + */ + public BitSet(long[] data) { + assert data.length > 0 : "data length is zero!"; + this.data = data; + } + + /** + * Sets the bit at specified index. + * + * @param index - position + */ + public void set(int index) { + data[index >>> 6] |= (1L << index); + } + + /** + * Returns true if the bit is set in the specified index. + * + * @param index - position + * @return - value at the bit position + */ + public boolean get(int index) { + return (data[index >>> 6] & (1L << index)) != 0; + } + + /** + * Number of bits + */ + public long bitSize() { + return (long) data.length * Long.SIZE; + } + + public long[] getData() { + return data; + } + + /** + * Combines the two BitArrays using bitwise OR. + */ + public void putAll(BitSet array) { + assert data.length == array.data.length : + "BitArrays must be of equal length (" + data.length + "!= " + array.data.length + ")"; + for (int i = 0; i < data.length; i++) { + data[i] |= array.data[i]; + } + } + + /** + * Clear the bit set. + */ + public void clear() { + Arrays.fill(data, 0); + } + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hive/common/util/BloomKFilter.java b/storage-api/hive-storage-api/src/java/org/apache/hive/common/util/BloomKFilter.java new file mode 100644 index 0000000000000000000000000000000000000000..45326ab009763c0560b474baba2d39ad77e13416 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hive/common/util/BloomKFilter.java @@ -0,0 +1,472 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.common.util; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; + +/** + * BloomKFilter is variation of {@link BloomFilter}. Unlike BloomFilter, BloomKFilter will spread + * 'k' hash bits within same cache line for better L1 cache performance. The way it works is, + * First hash code is computed from key which is used to locate the block offset (n-longs in bitset constitute a block) + * Subsequent 'k' hash codes are used to spread hash bits within the block. By default block size is chosen as 8, + * which is to match cache line size (8 longs = 64 bytes = cache line size). + * Refer {@link BloomKFilter#addBytes(byte[])} for more info. + * + * This implementation has much lesser L1 data cache misses than {@link BloomFilter}. + */ +public class BloomKFilter { + private byte[] BYTE_ARRAY_4 = new byte[4]; + private byte[] BYTE_ARRAY_8 = new byte[8]; + public static final float DEFAULT_FPP = 0.05f; + private static final int DEFAULT_BLOCK_SIZE = 8; + private static final int DEFAULT_BLOCK_SIZE_BITS = (int) (Math.log(DEFAULT_BLOCK_SIZE) / Math.log(2)); + private static final int DEFAULT_BLOCK_OFFSET_MASK = DEFAULT_BLOCK_SIZE - 1; + private static final int DEFAULT_BIT_OFFSET_MASK = Long.SIZE - 1; + private final long[] masks = new long[DEFAULT_BLOCK_SIZE]; + private BitSet bitSet; + private final int m; + private final int k; + // spread k-1 bits to adjacent longs, default is 8 + // spreading hash bits within blockSize * longs will make bloom filter L1 cache friendly + // default block size is set to 8 as most cache line sizes are 64 bytes and also AVX512 friendly + private final int totalBlockCount; + + static void checkArgument(boolean expression, String message) { + if (!expression) { + throw new IllegalArgumentException(message); + } + } + + public BloomKFilter(long maxNumEntries) { + checkArgument(maxNumEntries > 0, "expectedEntries should be > 0"); + long numBits = optimalNumOfBits(maxNumEntries, DEFAULT_FPP); + this.k = optimalNumOfHashFunctions(maxNumEntries, numBits); + int nLongs = (int) Math.ceil((double) numBits / (double) Long.SIZE); + // additional bits to pad long array to block size + int padLongs = DEFAULT_BLOCK_SIZE - nLongs % DEFAULT_BLOCK_SIZE; + this.m = (nLongs + padLongs) * Long.SIZE; + this.bitSet = new BitSet(m); + checkArgument((bitSet.data.length % DEFAULT_BLOCK_SIZE) == 0, "bitSet has to be block aligned"); + this.totalBlockCount = bitSet.data.length / DEFAULT_BLOCK_SIZE; + } + + /** + * A constructor to support rebuilding the BloomFilter from a serialized representation. + * @param bits + * @param numFuncs + */ + public BloomKFilter(long[] bits, int numFuncs) { + super(); + bitSet = new BitSet(bits); + this.m = bits.length * Long.SIZE; + this.k = numFuncs; + checkArgument((bitSet.data.length % DEFAULT_BLOCK_SIZE) == 0, "bitSet has to be block aligned"); + this.totalBlockCount = bitSet.data.length / DEFAULT_BLOCK_SIZE; + } + static int optimalNumOfHashFunctions(long n, long m) { + return Math.max(1, (int) Math.round((double) m / n * Math.log(2))); + } + + static long optimalNumOfBits(long n, double p) { + return (long) (-n * Math.log(p) / (Math.log(2) * Math.log(2))); + } + + public void add(byte[] val) { + addBytes(val); + } + + public void addBytes(byte[] val, int offset, int length) { + // We use the trick mentioned in "Less Hashing, Same Performance: Building a Better Bloom Filter" + // by Kirsch et.al. From abstract 'only two hash functions are necessary to effectively + // implement a Bloom filter without any loss in the asymptotic false positive probability' + + // Lets split up 64-bit hashcode into two 32-bit hash codes and employ the technique mentioned + // in the above paper + long hash64 = val == null ? Murmur3.NULL_HASHCODE : + Murmur3.hash64(val, offset, length); + addHash(hash64); + } + + public void addBytes(byte[] val) { + addBytes(val, 0, val.length); + } + + private void addHash(long hash64) { + final int hash1 = (int) hash64; + final int hash2 = (int) (hash64 >>> 32); + + int firstHash = hash1 + hash2; + // hashcode should be positive, flip all the bits if it's negative + if (firstHash < 0) { + firstHash = ~firstHash; + } + + // first hash is used to locate start of the block (blockBaseOffset) + // subsequent K hashes are used to generate K bits within a block of words + final int blockIdx = firstHash % totalBlockCount; + final int blockBaseOffset = blockIdx << DEFAULT_BLOCK_SIZE_BITS; + for (int i = 1; i <= k; i++) { + int combinedHash = hash1 + ((i + 1) * hash2); + // hashcode should be positive, flip all the bits if it's negative + if (combinedHash < 0) { + combinedHash = ~combinedHash; + } + // LSB 3 bits is used to locate offset within the block + final int absOffset = blockBaseOffset + (combinedHash & DEFAULT_BLOCK_OFFSET_MASK); + // Next 6 bits are used to locate offset within a long/word + final int bitPos = (combinedHash >>> DEFAULT_BLOCK_SIZE_BITS) & DEFAULT_BIT_OFFSET_MASK; + bitSet.data[absOffset] |= (1L << bitPos); + } + } + + public void addString(String val) { + addBytes(val.getBytes()); + } + + public void addByte(byte val) { + addBytes(new byte[]{val}); + } + + public void addInt(int val) { + // puts int in little endian order + addBytes(intToByteArrayLE(val)); + } + + + public void addLong(long val) { + // puts long in little endian order + addBytes(longToByteArrayLE(val)); + } + + public void addFloat(float val) { + addInt(Float.floatToIntBits(val)); + } + + public void addDouble(double val) { + addLong(Double.doubleToLongBits(val)); + } + + public boolean test(byte[] val) { + return testBytes(val); + } + + public boolean testBytes(byte[] val) { + return testBytes(val, 0, val.length); + } + + public boolean testBytes(byte[] val, int offset, int length) { + long hash64 = val == null ? Murmur3.NULL_HASHCODE : + Murmur3.hash64(val, offset, length); + return testHash(hash64); + } + + private boolean testHash(long hash64) { + final int hash1 = (int) hash64; + final int hash2 = (int) (hash64 >>> 32); + + int firstHash = hash1 + hash2; + // hashcode should be positive, flip all the bits if it's negative + if (firstHash < 0) { + firstHash = ~firstHash; + } + + // first hash is used to locate start of the block (blockBaseOffset) + // subsequent K hashes are used to generate K bits within a block of words + // To avoid branches during probe, a separate masks array is used for each longs/words within a block. + // data array and masks array are then traversed together and checked for corresponding set bits. + final int blockIdx = firstHash % totalBlockCount; + final int blockBaseOffset = blockIdx << DEFAULT_BLOCK_SIZE_BITS; + + // iterate and update masks array + for (int i = 1; i <= k; i++) { + int combinedHash = hash1 + ((i + 1) * hash2); + // hashcode should be positive, flip all the bits if it's negative + if (combinedHash < 0) { + combinedHash = ~combinedHash; + } + // LSB 3 bits is used to locate offset within the block + final int wordOffset = combinedHash & DEFAULT_BLOCK_OFFSET_MASK; + // Next 6 bits are used to locate offset within a long/word + final int bitPos = (combinedHash >>> DEFAULT_BLOCK_SIZE_BITS) & DEFAULT_BIT_OFFSET_MASK; + masks[wordOffset] |= (1L << bitPos); + } + + // traverse data and masks array together, check for set bits + long expected = 0; + for (int i = 0; i < DEFAULT_BLOCK_SIZE; i++) { + final long mask = masks[i]; + expected |= (bitSet.data[blockBaseOffset + i] & mask) ^ mask; + } + + // clear the mask for array reuse (this is to avoid masks array allocation in inner loop) + Arrays.fill(masks, 0); + + // if all bits are set, expected should be 0 + return expected == 0; + } + + public boolean testString(String val) { + return testBytes(val.getBytes()); + } + + public boolean testByte(byte val) { + return testBytes(new byte[]{val}); + } + + public boolean testInt(int val) { + return testBytes(intToByteArrayLE(val)); + } + + public boolean testLong(long val) { + return testBytes(longToByteArrayLE(val)); + } + + public boolean testFloat(float val) { + return testInt(Float.floatToIntBits(val)); + } + + public boolean testDouble(double val) { + return testLong(Double.doubleToLongBits(val)); + } + + private byte[] intToByteArrayLE(int val) { + BYTE_ARRAY_4[0] = (byte) (val >> 0); + BYTE_ARRAY_4[1] = (byte) (val >> 8); + BYTE_ARRAY_4[2] = (byte) (val >> 16); + BYTE_ARRAY_4[3] = (byte) (val >> 24); + return BYTE_ARRAY_4; + } + + private byte[] longToByteArrayLE(long val) { + BYTE_ARRAY_8[0] = (byte) (val >> 0); + BYTE_ARRAY_8[1] = (byte) (val >> 8); + BYTE_ARRAY_8[2] = (byte) (val >> 16); + BYTE_ARRAY_8[3] = (byte) (val >> 24); + BYTE_ARRAY_8[4] = (byte) (val >> 32); + BYTE_ARRAY_8[5] = (byte) (val >> 40); + BYTE_ARRAY_8[6] = (byte) (val >> 48); + BYTE_ARRAY_8[7] = (byte) (val >> 56); + return BYTE_ARRAY_8; + } + + public long sizeInBytes() { + return getBitSize() / 8; + } + + public int getBitSize() { + return bitSet.getData().length * Long.SIZE; + } + + public int getNumHashFunctions() { + return k; + } + + public int getNumBits() { + return m; + } + + public long[] getBitSet() { + return bitSet.getData(); + } + + @Override + public String toString() { + return "m: " + m + " k: " + k; + } + + /** + * Merge the specified bloom filter with current bloom filter. + * + * @param that - bloom filter to merge + */ + public void merge(BloomKFilter that) { + if (this != that && this.m == that.m && this.k == that.k) { + this.bitSet.putAll(that.bitSet); + } else { + throw new IllegalArgumentException("BloomKFilters are not compatible for merging." + + " this - " + this.toString() + " that - " + that.toString()); + } + } + + public void reset() { + this.bitSet.clear(); + } + + /** + * Serialize a bloom filter + * + * @param out output stream to write to + * @param bloomFilter BloomKFilter that needs to be seralized + */ + public static void serialize(OutputStream out, BloomKFilter bloomFilter) throws IOException { + /** + * Serialized BloomKFilter format: + * 1 byte for the number of hash functions. + * 1 big endian int(That is how OutputStream works) for the number of longs in the bitset + * big endina longs in the BloomKFilter bitset + */ + DataOutputStream dataOutputStream = new DataOutputStream(out); + dataOutputStream.writeByte(bloomFilter.k); + dataOutputStream.writeInt(bloomFilter.getBitSet().length); + for (long value : bloomFilter.getBitSet()) { + dataOutputStream.writeLong(value); + } + } + + /** + * Deserialize a bloom filter + * Read a byte stream, which was written by {@linkplain #serialize(OutputStream, BloomKFilter)} + * into a {@code BloomKFilter} + * + * @param in input bytestream + * @return deserialized BloomKFilter + */ + public static BloomKFilter deserialize(InputStream in) throws IOException { + if (in == null) { + throw new IOException("Input stream is null"); + } + + try { + DataInputStream dataInputStream = new DataInputStream(in); + int numHashFunc = dataInputStream.readByte(); + int bitsetArrayLen = dataInputStream.readInt(); + long[] data = new long[bitsetArrayLen]; + for (int i = 0; i < bitsetArrayLen; i++) { + data[i] = dataInputStream.readLong(); + } + return new BloomKFilter(data, numHashFunc); + } catch (RuntimeException e) { + IOException io = new IOException("Unable to deserialize BloomKFilter"); + io.initCause(e); + throw io; + } + } + + // Given a byte array consisting of a serialized BloomKFilter, gives the offset (from 0) + // for the start of the serialized long values that make up the bitset. + // NumHashFunctions (1 byte) + bitset array length (4 bytes) + public static final int START_OF_SERIALIZED_LONGS = 5; + + /** + * Merges BloomKFilter bf2 into bf1. + * Assumes 2 BloomKFilters with the same size/hash functions are serialized to byte arrays + * + * @param bf1Bytes + * @param bf1Start + * @param bf1Length + * @param bf2Bytes + * @param bf2Start + * @param bf2Length + */ + public static void mergeBloomFilterBytes( + byte[] bf1Bytes, int bf1Start, int bf1Length, + byte[] bf2Bytes, int bf2Start, int bf2Length) { + if (bf1Length != bf2Length) { + throw new IllegalArgumentException("bf1Length " + bf1Length + " does not match bf2Length " + bf2Length); + } + + // Validation on the bitset size/3 hash functions. + for (int idx = 0; idx < START_OF_SERIALIZED_LONGS; ++idx) { + if (bf1Bytes[bf1Start + idx] != bf2Bytes[bf2Start + idx]) { + throw new IllegalArgumentException("bf1 NumHashFunctions/NumBits does not match bf2"); + } + } + + // Just bitwise-OR the bits together - size/# functions should be the same, + // rest of the data is serialized long values for the bitset which are supposed to be bitwise-ORed. + for (int idx = START_OF_SERIALIZED_LONGS; idx < bf1Length; ++idx) { + bf1Bytes[bf1Start + idx] |= bf2Bytes[bf2Start + idx]; + } + } + + /** + * Bare metal bit set implementation. For performance reasons, this implementation does not check + * for index bounds nor expand the bit set size if the specified index is greater than the size. + */ + public static class BitSet { + private final long[] data; + + public BitSet(long bits) { + this(new long[(int) Math.ceil((double) bits / (double) Long.SIZE)]); + } + + /** + * Deserialize long array as bit set. + * + * @param data - bit array + */ + public BitSet(long[] data) { + assert data.length > 0 : "data length is zero!"; + this.data = data; + } + + /** + * Sets the bit at specified index. + * + * @param index - position + */ + public void set(int index) { + data[index >>> 6] |= (1L << index); + } + + /** + * Returns true if the bit is set in the specified index. + * + * @param index - position + * @return - value at the bit position + */ + public boolean get(int index) { + return (data[index >>> 6] & (1L << index)) != 0; + } + + /** + * Number of bits + */ + public int bitSize() { + return data.length * Long.SIZE; + } + + public long[] getData() { + return data; + } + + /** + * Combines the two BitArrays using bitwise OR. + */ + public void putAll(BloomKFilter.BitSet array) { + assert data.length == array.data.length : + "BitArrays must be of equal length (" + data.length + "!= " + array.data.length + ")"; + for (int i = 0; i < data.length; i++) { + data[i] |= array.data[i]; + } + } + + /** + * Clear the bit set. + */ + public void clear() { + Arrays.fill(data, 0); + } + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java b/storage-api/hive-storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..727c1e6699479577cbcbee22d7bee13a6f299c02 --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.common.util; + +import java.math.BigDecimal; +import java.text.SimpleDateFormat; + +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; + + +/** + * DateUtils. Thread-safe class + * + */ +public class IntervalDayTimeUtils { + + private static final ThreadLocal dateFormatLocal = new ThreadLocal() { + @Override + protected SimpleDateFormat initialValue() { + return new SimpleDateFormat("yyyy-MM-dd"); + } + }; + + public static SimpleDateFormat getDateFormat() { + return dateFormatLocal.get(); + } + + public static final int NANOS_PER_SEC = 1000000000; + public static final BigDecimal MAX_INT_BD = new BigDecimal(Integer.MAX_VALUE); + public static final BigDecimal NANOS_PER_SEC_BD = new BigDecimal(NANOS_PER_SEC); + + public static int parseNumericValueWithRange(String fieldName, + String strVal, int minValue, int maxValue) throws IllegalArgumentException { + int result = 0; + if (strVal != null) { + result = Integer.parseInt(strVal); + if (result < minValue || result > maxValue) { + throw new IllegalArgumentException(String.format("%s value %d outside range [%d, %d]", + fieldName, result, minValue, maxValue)); + } + } + return result; + } + + public static long getIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime) { + return intervalDayTime.getTotalSeconds() * NANOS_PER_SEC + intervalDayTime.getNanos(); + } + + public static void setIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime, + long totalNanos) { + intervalDayTime.set(totalNanos / NANOS_PER_SEC, (int) (totalNanos % NANOS_PER_SEC)); + } + + public static long getIntervalDayTimeTotalSecondsFromTotalNanos(long totalNanos) { + return totalNanos / NANOS_PER_SEC; + } + + public static int getIntervalDayTimeNanosFromTotalNanos(long totalNanos) { + return (int) (totalNanos % NANOS_PER_SEC); + } +} diff --git a/storage-api/hive-storage-api/src/java/org/apache/hive/common/util/Murmur3.java b/storage-api/hive-storage-api/src/java/org/apache/hive/common/util/Murmur3.java new file mode 100644 index 0000000000000000000000000000000000000000..eb050820d67a8606e10d190cb797aedbeb219f7e --- /dev/null +++ b/storage-api/hive-storage-api/src/java/org/apache/hive/common/util/Murmur3.java @@ -0,0 +1,431 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.common.util; + +/** + * Murmur3 is successor to Murmur2 fast non-crytographic hash algorithms. + * + * Murmur3 32 and 128 bit variants. + * 32-bit Java port of https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp#94 + * 128-bit Java port of https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp#255 + * + * This is a public domain code with no copyrights. + * From homepage of MurmurHash (https://code.google.com/p/smhasher/), + * "All MurmurHash versions are public domain software, and the author disclaims all copyright + * to their code." + */ +public class Murmur3 { + // from 64-bit linear congruential generator + public static final long NULL_HASHCODE = 2862933555777941757L; + + // Constants for 32 bit variant + private static final int C1_32 = 0xcc9e2d51; + private static final int C2_32 = 0x1b873593; + private static final int R1_32 = 15; + private static final int R2_32 = 13; + private static final int M_32 = 5; + private static final int N_32 = 0xe6546b64; + + // Constants for 128 bit variant + private static final long C1 = 0x87c37b91114253d5L; + private static final long C2 = 0x4cf5ad432745937fL; + private static final int R1 = 31; + private static final int R2 = 27; + private static final int R3 = 33; + private static final int M = 5; + private static final int N1 = 0x52dce729; + private static final int N2 = 0x38495ab5; + + public static final int DEFAULT_SEED = 104729; + + /** + * Murmur3 32-bit variant. + * + * @param data - input byte array + * @return - hashcode + */ + public static int hash32(byte[] data) { + return hash32(data, data.length, DEFAULT_SEED); + } + + /** + * Murmur3 32-bit variant. + * + * @param data - input byte array + * @param length - length of array + * @param seed - seed. (default 0) + * @return - hashcode + */ + public static int hash32(byte[] data, int length, int seed) { + int hash = seed; + final int nblocks = length >> 2; + + // body + for (int i = 0; i < nblocks; i++) { + int i_4 = i << 2; + int k = (data[i_4] & 0xff) + | ((data[i_4 + 1] & 0xff) << 8) + | ((data[i_4 + 2] & 0xff) << 16) + | ((data[i_4 + 3] & 0xff) << 24); + + // mix functions + k *= C1_32; + k = Integer.rotateLeft(k, R1_32); + k *= C2_32; + hash ^= k; + hash = Integer.rotateLeft(hash, R2_32) * M_32 + N_32; + } + + // tail + int idx = nblocks << 2; + int k1 = 0; + switch (length - idx) { + case 3: + k1 ^= data[idx + 2] << 16; + case 2: + k1 ^= data[idx + 1] << 8; + case 1: + k1 ^= data[idx]; + + // mix functions + k1 *= C1_32; + k1 = Integer.rotateLeft(k1, R1_32); + k1 *= C2_32; + hash ^= k1; + } + + // finalization + hash ^= length; + hash ^= (hash >>> 16); + hash *= 0x85ebca6b; + hash ^= (hash >>> 13); + hash *= 0xc2b2ae35; + hash ^= (hash >>> 16); + + return hash; + } + + /** + * Murmur3 64-bit variant. This is essentially MSB 8 bytes of Murmur3 128-bit variant. + * + * @param data - input byte array + * @return - hashcode + */ + public static long hash64(byte[] data) { + return hash64(data, 0, data.length, DEFAULT_SEED); + } + + public static long hash64(byte[] data, int offset, int length) { + return hash64(data, offset, length, DEFAULT_SEED); + } + + /** + * Murmur3 64-bit variant. This is essentially MSB 8 bytes of Murmur3 128-bit variant. + * + * @param data - input byte array + * @param length - length of array + * @param seed - seed. (default is 0) + * @return - hashcode + */ + public static long hash64(byte[] data, int offset, int length, int seed) { + long hash = seed; + final int nblocks = length >> 3; + + // body + for (int i = 0; i < nblocks; i++) { + final int i8 = i << 3; + long k = ((long) data[offset + i8] & 0xff) + | (((long) data[offset + i8 + 1] & 0xff) << 8) + | (((long) data[offset + i8 + 2] & 0xff) << 16) + | (((long) data[offset + i8 + 3] & 0xff) << 24) + | (((long) data[offset + i8 + 4] & 0xff) << 32) + | (((long) data[offset + i8 + 5] & 0xff) << 40) + | (((long) data[offset + i8 + 6] & 0xff) << 48) + | (((long) data[offset + i8 + 7] & 0xff) << 56); + + // mix functions + k *= C1; + k = Long.rotateLeft(k, R1); + k *= C2; + hash ^= k; + hash = Long.rotateLeft(hash, R2) * M + N1; + } + + // tail + long k1 = 0; + int tailStart = nblocks << 3; + switch (length - tailStart) { + case 7: + k1 ^= ((long) data[offset + tailStart + 6] & 0xff) << 48; + case 6: + k1 ^= ((long) data[offset + tailStart + 5] & 0xff) << 40; + case 5: + k1 ^= ((long) data[offset + tailStart + 4] & 0xff) << 32; + case 4: + k1 ^= ((long) data[offset + tailStart + 3] & 0xff) << 24; + case 3: + k1 ^= ((long) data[offset + tailStart + 2] & 0xff) << 16; + case 2: + k1 ^= ((long) data[offset + tailStart + 1] & 0xff) << 8; + case 1: + k1 ^= ((long) data[offset + tailStart] & 0xff); + k1 *= C1; + k1 = Long.rotateLeft(k1, R1); + k1 *= C2; + hash ^= k1; + } + + // finalization + hash ^= length; + hash = fmix64(hash); + + return hash; + } + + /** + * Murmur3 128-bit variant. + * + * @param data - input byte array + * @return - hashcode (2 longs) + */ + public static long[] hash128(byte[] data) { + return hash128(data, 0, data.length, DEFAULT_SEED); + } + + /** + * Murmur3 128-bit variant. + * + * @param data - input byte array + * @param offset - the first element of array + * @param length - length of array + * @param seed - seed. (default is 0) + * @return - hashcode (2 longs) + */ + public static long[] hash128(byte[] data, int offset, int length, int seed) { + long h1 = seed; + long h2 = seed; + final int nblocks = length >> 4; + + // body + for (int i = 0; i < nblocks; i++) { + final int i16 = i << 4; + long k1 = ((long) data[offset + i16] & 0xff) + | (((long) data[offset + i16 + 1] & 0xff) << 8) + | (((long) data[offset + i16 + 2] & 0xff) << 16) + | (((long) data[offset + i16 + 3] & 0xff) << 24) + | (((long) data[offset + i16 + 4] & 0xff) << 32) + | (((long) data[offset + i16 + 5] & 0xff) << 40) + | (((long) data[offset + i16 + 6] & 0xff) << 48) + | (((long) data[offset + i16 + 7] & 0xff) << 56); + + long k2 = ((long) data[offset + i16 + 8] & 0xff) + | (((long) data[offset + i16 + 9] & 0xff) << 8) + | (((long) data[offset + i16 + 10] & 0xff) << 16) + | (((long) data[offset + i16 + 11] & 0xff) << 24) + | (((long) data[offset + i16 + 12] & 0xff) << 32) + | (((long) data[offset + i16 + 13] & 0xff) << 40) + | (((long) data[offset + i16 + 14] & 0xff) << 48) + | (((long) data[offset + i16 + 15] & 0xff) << 56); + + // mix functions for k1 + k1 *= C1; + k1 = Long.rotateLeft(k1, R1); + k1 *= C2; + h1 ^= k1; + h1 = Long.rotateLeft(h1, R2); + h1 += h2; + h1 = h1 * M + N1; + + // mix functions for k2 + k2 *= C2; + k2 = Long.rotateLeft(k2, R3); + k2 *= C1; + h2 ^= k2; + h2 = Long.rotateLeft(h2, R1); + h2 += h1; + h2 = h2 * M + N2; + } + + // tail + long k1 = 0; + long k2 = 0; + int tailStart = nblocks << 4; + switch (length - tailStart) { + case 15: + k2 ^= (long) (data[offset + tailStart + 14] & 0xff) << 48; + case 14: + k2 ^= (long) (data[offset + tailStart + 13] & 0xff) << 40; + case 13: + k2 ^= (long) (data[offset + tailStart + 12] & 0xff) << 32; + case 12: + k2 ^= (long) (data[offset + tailStart + 11] & 0xff) << 24; + case 11: + k2 ^= (long) (data[offset + tailStart + 10] & 0xff) << 16; + case 10: + k2 ^= (long) (data[offset + tailStart + 9] & 0xff) << 8; + case 9: + k2 ^= (long) (data[offset + tailStart + 8] & 0xff); + k2 *= C2; + k2 = Long.rotateLeft(k2, R3); + k2 *= C1; + h2 ^= k2; + + case 8: + k1 ^= (long) (data[offset + tailStart + 7] & 0xff) << 56; + case 7: + k1 ^= (long) (data[offset + tailStart + 6] & 0xff) << 48; + case 6: + k1 ^= (long) (data[offset + tailStart + 5] & 0xff) << 40; + case 5: + k1 ^= (long) (data[offset + tailStart + 4] & 0xff) << 32; + case 4: + k1 ^= (long) (data[offset + tailStart + 3] & 0xff) << 24; + case 3: + k1 ^= (long) (data[offset + tailStart + 2] & 0xff) << 16; + case 2: + k1 ^= (long) (data[offset + tailStart + 1] & 0xff) << 8; + case 1: + k1 ^= (long) (data[offset + tailStart] & 0xff); + k1 *= C1; + k1 = Long.rotateLeft(k1, R1); + k1 *= C2; + h1 ^= k1; + } + + // finalization + h1 ^= length; + h2 ^= length; + + h1 += h2; + h2 += h1; + + h1 = fmix64(h1); + h2 = fmix64(h2); + + h1 += h2; + h2 += h1; + + return new long[]{h1, h2}; + } + + private static long fmix64(long h) { + h ^= (h >>> 33); + h *= 0xff51afd7ed558ccdL; + h ^= (h >>> 33); + h *= 0xc4ceb9fe1a85ec53L; + h ^= (h >>> 33); + return h; + } + + public static class IncrementalHash32 { + byte[] tail = new byte[3]; + int tailLen; + int totalLen; + int hash; + + public final void start(int hash) { + tailLen = totalLen = 0; + this.hash = hash; + } + + public final void add(byte[] data, int offset, int length) { + if (length == 0) return; + totalLen += length; + if (tailLen + length < 4) { + System.arraycopy(data, offset, tail, tailLen, length); + tailLen += length; + return; + } + int offset2 = 0; + if (tailLen > 0) { + offset2 = (4 - tailLen); + int k = -1; + switch (tailLen) { + case 1: + k = orBytes(tail[0], data[offset], data[offset + 1], data[offset + 2]); + break; + case 2: + k = orBytes(tail[0], tail[1], data[offset], data[offset + 1]); + break; + case 3: + k = orBytes(tail[0], tail[1], tail[2], data[offset]); + break; + default: throw new AssertionError(tailLen); + } + // mix functions + k *= C1_32; + k = Integer.rotateLeft(k, R1_32); + k *= C2_32; + hash ^= k; + hash = Integer.rotateLeft(hash, R2_32) * M_32 + N_32; + } + int length2 = length - offset2; + offset += offset2; + final int nblocks = length2 >> 2; + + for (int i = 0; i < nblocks; i++) { + int i_4 = (i << 2) + offset; + int k = orBytes(data[i_4], data[i_4 + 1], data[i_4 + 2], data[i_4 + 3]); + + // mix functions + k *= C1_32; + k = Integer.rotateLeft(k, R1_32); + k *= C2_32; + hash ^= k; + hash = Integer.rotateLeft(hash, R2_32) * M_32 + N_32; + } + + int consumed = (nblocks << 2); + tailLen = length2 - consumed; + if (consumed == length2) return; + System.arraycopy(data, offset + consumed, tail, 0, tailLen); + } + + public final int end() { + int k1 = 0; + switch (tailLen) { + case 3: + k1 ^= tail[2] << 16; + case 2: + k1 ^= tail[1] << 8; + case 1: + k1 ^= tail[0]; + + // mix functions + k1 *= C1_32; + k1 = Integer.rotateLeft(k1, R1_32); + k1 *= C2_32; + hash ^= k1; + } + + // finalization + hash ^= totalLen; + hash ^= (hash >>> 16); + hash *= 0x85ebca6b; + hash ^= (hash >>> 13); + hash *= 0xc2b2ae35; + hash ^= (hash >>> 16); + return hash; + } + } + + private static int orBytes(byte b1, byte b2, byte b3, byte b4) { + return (b1 & 0xff) | ((b2 & 0xff) << 8) | ((b3 & 0xff) << 16) | ((b4 & 0xff) << 24); + } +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/VersionTestBase.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/VersionTestBase.java new file mode 100644 index 0000000000000000000000000000000000000000..489496e418a6fede80026357f49b50f2dc166e86 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/VersionTestBase.java @@ -0,0 +1,322 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; + +import org.junit.*; + +import static org.junit.Assert.*; + +public class VersionTestBase { + + public String getParameterTypeName(Class parameterType, + Map versionedClassToNameMap) { + if (versionedClassToNameMap.containsKey(parameterType)) { + return versionedClassToNameMap.get(parameterType); + } else { + return parameterType.getSimpleName(); + } + } + + public String getMethodKey(Method method, Map versionedClassToNameMap) { + //------------------------------------------------- + StringBuilder sb = new StringBuilder(); + + int modifiers = method.getModifiers(); + if ((modifiers & Modifier.STATIC) != 0) { + sb.append("static"); + } else { + sb.append("non-static"); + } + sb.append(" "); + Class returnType = method.getReturnType(); + sb.append(getParameterTypeName(returnType, versionedClassToNameMap)); + sb.append(" "); + sb.append(method.getName()); + Class[] parameterTypes = method.getParameterTypes(); + sb.append("("); + boolean isFirst = true; + for (Class parameterType : parameterTypes) { + if (!isFirst) { + sb.append(", "); + } + sb.append(getParameterTypeName(parameterType, versionedClassToNameMap)); + isFirst = false; + } + sb.append(")"); + Class[] exceptionsThrown = method.getExceptionTypes(); + if (exceptionsThrown.length > 0) { + sb.append(" throws "); + isFirst = true; + for (Class exceptionThrown : exceptionsThrown) { + if (!isFirst) { + sb.append(", "); + } + sb.append(exceptionThrown.getSimpleName()); + isFirst = false; + } + } + + return sb.toString(); + //------------------------------------------------- + } + + public String getFieldKey(Field field, Map versionedClassToNameMap) throws IllegalAccessException { + //------------------------------------------------- + StringBuilder sb = new StringBuilder(); + + int modifiers = field.getModifiers(); + if ((modifiers & Modifier.STATIC) != 0) { + sb.append("static"); + } else { + sb.append("non-static"); + } + sb.append(" "); + Class fieldType = field.getType(); + sb.append(getParameterTypeName(fieldType, versionedClassToNameMap)); + sb.append(" "); + sb.append(field.getName()); + if ((modifiers & Modifier.STATIC) != 0) { + sb.append(" "); + sb.append(field.get(null)); + } + + return sb.toString(); + //------------------------------------------------- + } + + public Method[] onlyPublicMethods(Method[] methods) { + List resultList = new ArrayList(); + for (Method method : methods) { + if ((method.getModifiers() & Modifier.PUBLIC) != 0) { + resultList.add(method); + } + } + return resultList.toArray(new Method[0]); + } + + public Field[] onlyPublicFields(Field[] fields) { + List resultList = new ArrayList(); + for (Field field : fields) { + if ((field.getModifiers() & Modifier.PUBLIC) != 0) { + resultList.add(field); + } + } + return resultList.toArray(new Field[0]); + } + + public TreeSet getMethodKeySetForAnnotation(Method[] methods, Class annotationClass, + Map versionedClassToNameMap) + throws IllegalAccessException { + TreeSet result = new TreeSet(); + + for (Method method : methods) { + Annotation[] declaredAnnotations = method.getDeclaredAnnotations(); + boolean isFound = false; + for (Annotation declaredAnnotation : declaredAnnotations) { + if (declaredAnnotation.annotationType().equals(annotationClass)) { + isFound = true; + break; + } + } + if (!isFound) { + continue; + } + result.add(getMethodKey(method, versionedClassToNameMap)); + } + return result; + } + + public TreeSet getMethodKeySetExcludingAnnotations(Method[] methods, + List versionAnnotations, Map versionedClassToNameMap) + throws IllegalAccessException { + TreeSet result = new TreeSet(); + + for (Method method : methods) { + Annotation[] declaredAnnotations = method.getDeclaredAnnotations(); + boolean isFound = false; + for (Annotation declaredAnnotation : declaredAnnotations) { + for (Class versionAnnotation : versionAnnotations) { + if (declaredAnnotation.annotationType().equals(versionAnnotation)) { + isFound = true; + break; + } + } + if (isFound) { + break; + } + } + if (isFound) { + continue; + } + String methodKey = getMethodKey(method, versionedClassToNameMap); + if (!methodKey.equals("non-static int compareTo(Object)")) { + result.add(methodKey); + } + } + return result; + } + + public TreeSet getFieldKeySetForAnnotation(Field[] fields, + Class annotationClass, Map versionedClassToNameMap) + throws IllegalAccessException { + TreeSet result = new TreeSet(); + + for (Field field : fields) { + Annotation[] declaredAnnotations = field.getDeclaredAnnotations(); + boolean isFound = false; + for (Annotation declaredAnnotation : declaredAnnotations) { + if (declaredAnnotation.annotationType().equals(annotationClass)) { + isFound = true; + break; + } + } + if (!isFound) { + continue; + } + result.add(getFieldKey(field, versionedClassToNameMap)); + } + return result; + } + + public TreeSet getFieldKeySetExcludingAnnotations(Field[] fields, + List versionAnnotations, Map versionedClassToNameMap) + throws IllegalAccessException { + TreeSet result = new TreeSet(); + + for (Field field : fields) { + Annotation[] declaredAnnotations = field.getDeclaredAnnotations(); + boolean isFound = false; + for (Annotation declaredAnnotation : declaredAnnotations) { + for (Class versionAnnotation : versionAnnotations) { + if (declaredAnnotation.annotationType().equals(versionAnnotation)) { + isFound = true; + break; + } + } + if (isFound) { + break; + } + } + if (isFound) { + continue; + } + result.add(getFieldKey(field, versionedClassToNameMap)); + } + return result; + } + + // For now, olderClass has 1 version and newerClass 2 versions... + public void doVerifyVersions( + Class olderClass, Class olderVersionClass, + Class newerClass, Class newerVersionClass, + Map versionedClassToNameMap) throws IllegalAccessException { + + List olderVersionClasses = new ArrayList(); + olderVersionClasses.add(olderVersionClass); + + List newerVersionClasses = new ArrayList(); + newerVersionClasses.add(olderVersionClass); + newerVersionClasses.add(newerVersionClass); + + //---------------------------------------------------------------------------------------------- + Method[] olderMethods = onlyPublicMethods(olderClass.getDeclaredMethods()); + TreeSet olderMethodSet = + getMethodKeySetForAnnotation(olderMethods, olderVersionClass, versionedClassToNameMap); + + TreeSet olderNoMethodAnnotationsSet = + getMethodKeySetExcludingAnnotations(olderMethods, olderVersionClasses, versionedClassToNameMap); + + Field[] olderFields = onlyPublicFields(olderClass.getFields()); + TreeSet olderFieldSet = + getFieldKeySetForAnnotation(olderFields, olderVersionClass, versionedClassToNameMap); + + TreeSet olderNoFieldAnnotationsSet = + getFieldKeySetExcludingAnnotations(olderFields, olderVersionClasses, versionedClassToNameMap); + //---------------------------------------------------------------------------------------------- + + Method[] newerMethods = onlyPublicMethods(newerClass.getDeclaredMethods()); + TreeSet newerMethodSetV1 = + getMethodKeySetForAnnotation(newerMethods, olderVersionClass, versionedClassToNameMap); + TreeSet newerMethodSetV2 = + getMethodKeySetForAnnotation(newerMethods, newerVersionClass, versionedClassToNameMap); + + TreeSet newerNoMethodAnnotationsSetV1andV2 = + getMethodKeySetExcludingAnnotations(newerMethods, newerVersionClasses, versionedClassToNameMap); + + Field[] newerFields = onlyPublicFields(newerClass.getFields()); + // doDisplayFields(newerFields, newerClass); + TreeSet newerFieldSetV1 = + getFieldKeySetForAnnotation(newerFields, olderVersionClass, versionedClassToNameMap); + TreeSet newerFieldSetV2 = + getFieldKeySetForAnnotation(newerFields, newerVersionClass, versionedClassToNameMap); + + TreeSet newerNoFieldAnnotationsSetV1andV2 = + getFieldKeySetExcludingAnnotations(newerFields, newerVersionClasses, versionedClassToNameMap); + + //---------------------------------------------------------------------------------------------- + // VALIDATION + //---------------------------------------------------------------------------------------------- + + // No version annotation? + if (olderNoMethodAnnotationsSet.size() != 0) { + Assert.assertTrue("Class " + olderClass.getSimpleName() + " has 1 or more public methods without a version V1 annotation " + + olderNoMethodAnnotationsSet.toString(), false); + } + if (olderNoFieldAnnotationsSet.size() != 0) { + Assert.assertTrue("Class " + olderClass.getSimpleName() + " has 1 or more public fields without a version V1 annotation " + + olderNoFieldAnnotationsSet.toString(), false); + } + if (newerNoMethodAnnotationsSetV1andV2.size() != 0) { + Assert.assertTrue("Class " + newerClass.getSimpleName() + " has 1 or more public methods without a version V1 or V2 annotation " + + newerNoMethodAnnotationsSetV1andV2.toString(), false); + } + if (newerNoFieldAnnotationsSetV1andV2.size() != 0) { + Assert.assertTrue("Class " + newerClass.getSimpleName() + " has 1 or more public methods without a version V1 or V2 annotation " + + newerNoFieldAnnotationsSetV1andV2.toString(), false); + } + + // Do the V1 methods of older and newer match? + if (!olderMethodSet.equals(newerMethodSetV1)) { + TreeSet leftCopy = new TreeSet(olderMethodSet); + leftCopy.removeAll(newerMethodSetV1); + TreeSet rightCopy = new TreeSet(newerMethodSetV1); + rightCopy.removeAll(olderMethodSet); + Assert.assertTrue("Class " + olderClass.getSimpleName() + " and class " + newerClass.getSimpleName() + " methods are different for V1 " + + leftCopy.toString() + " " + rightCopy.toString(), false); + } + + // Do the V1 fields of older and newer match? + if (!olderFieldSet.equals(newerFieldSetV1)) { + TreeSet leftCopy = new TreeSet(olderFieldSet); + leftCopy.removeAll(newerFieldSetV1); + TreeSet rightCopy = new TreeSet(newerFieldSetV1); + rightCopy.removeAll(olderFieldSet); + Assert.assertTrue("Class " + olderClass.getSimpleName() + " and class " + newerClass.getSimpleName() + " fields are different for V1 " + + leftCopy.toString() + " " + rightCopy.toString(), false); + } + } +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorTxnList.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorTxnList.java new file mode 100644 index 0000000000000000000000000000000000000000..867b652116101fb784ccb30a267a6f4306761cd1 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorTxnList.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.BitSet; + +public class TestValidCompactorTxnList { + + @Test + public void minTxnHigh() { + BitSet bitSet = new BitSet(2); + bitSet.set(0, 2); + ValidTxnList txns = new ValidCompactorTxnList(new long[]{3, 4}, bitSet, 2); + ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9); + Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp); + } + + @Test + public void maxTxnLow() { + BitSet bitSet = new BitSet(2); + bitSet.set(0, 2); + ValidTxnList txns = new ValidCompactorTxnList(new long[]{13, 14}, bitSet, 12); + ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9); + Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp); + } + + @Test + public void minTxnHighNoExceptions() { + ValidTxnList txns = new ValidCompactorTxnList(new long[0], new BitSet(), 5); + ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9); + Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp); + } + + @Test + public void maxTxnLowNoExceptions() { + ValidTxnList txns = new ValidCompactorTxnList(new long[0], new BitSet(), 15); + ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9); + Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp); + } + + @Test + public void exceptionsAllBelow() { + BitSet bitSet = new BitSet(2); + bitSet.set(0, 2); + ValidTxnList txns = new ValidCompactorTxnList(new long[]{3, 6}, bitSet, 3); + ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9); + Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp); + } + + @Test + public void exceptionsInMidst() { + BitSet bitSet = new BitSet(1); + bitSet.set(0, 1); + ValidTxnList txns = new ValidCompactorTxnList(new long[]{8}, bitSet, 7); + ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9); + Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp); + } + @Test + public void exceptionsAbveHighWaterMark() { + BitSet bitSet = new BitSet(4); + bitSet.set(0, 4); + ValidTxnList txns = new ValidCompactorTxnList(new long[]{8, 11, 17, 29}, bitSet, 15); + Assert.assertArrayEquals("", new long[]{8, 11}, txns.getInvalidTransactions()); + ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9); + Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp); + rsp = txns.isTxnRangeValid(12, 16); + Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp); + } + + @Test + public void writeToString() { + BitSet bitSet = new BitSet(4); + bitSet.set(0, 4); + ValidTxnList txns = new ValidCompactorTxnList(new long[]{7, 9, 10, Long.MAX_VALUE}, bitSet, 8); + Assert.assertEquals("8:" + Long.MAX_VALUE + "::7", txns.writeToString()); + txns = new ValidCompactorTxnList(); + Assert.assertEquals(Long.toString(Long.MAX_VALUE) + ":" + Long.MAX_VALUE + "::", txns.writeToString()); + txns = new ValidCompactorTxnList(new long[0], new BitSet(), 23); + Assert.assertEquals("23:" + Long.MAX_VALUE + "::", txns.writeToString()); + } + + @Test + public void readFromString() { + ValidCompactorTxnList txns = new ValidCompactorTxnList("37:" + Long.MAX_VALUE + "::7,9,10"); + Assert.assertEquals(37L, txns.getHighWatermark()); + Assert.assertNull(txns.getMinOpenTxn()); + Assert.assertArrayEquals(new long[]{7L, 9L, 10L}, txns.getInvalidTransactions()); + txns = new ValidCompactorTxnList("21:" + Long.MAX_VALUE + ":"); + Assert.assertEquals(21L, txns.getHighWatermark()); + Assert.assertNull(txns.getMinOpenTxn()); + Assert.assertEquals(0, txns.getInvalidTransactions().length); + } + + @Test + public void testAbortedTxn() throws Exception { + ValidCompactorTxnList txnList = new ValidCompactorTxnList("5:4::1,2,3"); + Assert.assertEquals(5L, txnList.getHighWatermark()); + Assert.assertEquals(4, txnList.getMinOpenTxn().longValue()); + Assert.assertArrayEquals(new long[]{1L, 2L, 3L}, txnList.getInvalidTransactions()); + } + + @Test + public void testAbortedRange() throws Exception { + ValidCompactorTxnList txnList = new ValidCompactorTxnList("11:4::5,6,7,8"); + ValidTxnList.RangeResponse rsp = txnList.isTxnRangeAborted(1L, 3L); + Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp); + rsp = txnList.isTxnRangeAborted(9L, 10L); + Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp); + rsp = txnList.isTxnRangeAborted(6L, 7L); + Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp); + rsp = txnList.isTxnRangeAborted(4L, 6L); + Assert.assertEquals(ValidTxnList.RangeResponse.SOME, rsp); + rsp = txnList.isTxnRangeAborted(6L, 13L); + Assert.assertEquals(ValidTxnList.RangeResponse.SOME, rsp); + } +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java new file mode 100644 index 0000000000000000000000000000000000000000..00ee820b2ebc4111b4b43fd48e8dd75978ee0754 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common; + +import junit.framework.Assert; +import org.apache.hadoop.conf.Configuration; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.util.BitSet; + +/** + * Tests for {@link ValidReadTxnList} + */ +public class TestValidReadTxnList { + + @Test + public void noExceptions() throws Exception { + ValidTxnList txnList = new ValidReadTxnList(new long[0], new BitSet(), 1, Long.MAX_VALUE); + String str = txnList.writeToString(); + Assert.assertEquals("1:" + Long.MAX_VALUE + "::", str); + ValidTxnList newList = new ValidReadTxnList(); + newList.readFromString(str); + Assert.assertTrue(newList.isTxnValid(1)); + Assert.assertFalse(newList.isTxnValid(2)); + } + + @Test + public void exceptions() throws Exception { + ValidTxnList txnList = new ValidReadTxnList(new long[]{2L,4L}, new BitSet(), 5, 4L); + String str = txnList.writeToString(); + Assert.assertEquals("5:4:2,4:", str); + ValidTxnList newList = new ValidReadTxnList(); + newList.readFromString(str); + Assert.assertTrue(newList.isTxnValid(1)); + Assert.assertFalse(newList.isTxnValid(2)); + Assert.assertTrue(newList.isTxnValid(3)); + Assert.assertFalse(newList.isTxnValid(4)); + Assert.assertTrue(newList.isTxnValid(5)); + Assert.assertFalse(newList.isTxnValid(6)); + } + + @Test + public void longEnoughToCompress() throws Exception { + long[] exceptions = new long[1000]; + for (int i = 0; i < 1000; i++) exceptions[i] = i + 100; + ValidTxnList txnList = new ValidReadTxnList(exceptions, new BitSet(), 2000, 900); + String str = txnList.writeToString(); + ValidTxnList newList = new ValidReadTxnList(); + newList.readFromString(str); + for (int i = 0; i < 100; i++) Assert.assertTrue(newList.isTxnValid(i)); + for (int i = 100; i < 1100; i++) Assert.assertFalse(newList.isTxnValid(i)); + for (int i = 1100; i < 2001; i++) Assert.assertTrue(newList.isTxnValid(i)); + Assert.assertFalse(newList.isTxnValid(2001)); + } + + @Test + public void readWriteConfig() throws Exception { + long[] exceptions = new long[1000]; + for (int i = 0; i < 1000; i++) exceptions[i] = i + 100; + ValidTxnList txnList = new ValidReadTxnList(exceptions, new BitSet(), 2000, 900); + String str = txnList.writeToString(); + Configuration conf = new Configuration(); + conf.set(ValidTxnList.VALID_TXNS_KEY, str); + File tmpFile = File.createTempFile("TestValidTxnImpl", "readWriteConfig"); + DataOutputStream out = new DataOutputStream(new FileOutputStream(tmpFile)); + conf.write(out); + out.close(); + DataInputStream in = new DataInputStream(new FileInputStream(tmpFile)); + Configuration newConf = new Configuration(); + newConf.readFields(in); + Assert.assertEquals(str, newConf.get(ValidTxnList.VALID_TXNS_KEY)); + } + + @Test + public void testAbortedTxn() throws Exception { + long[] exceptions = {2L, 4L, 6L, 8L, 10L}; + BitSet bitSet = new BitSet(exceptions.length); + bitSet.set(0); // mark txn "2L" aborted + bitSet.set(3); // mark txn "8L" aborted + ValidTxnList txnList = new ValidReadTxnList(exceptions, bitSet, 11, 4L); + String str = txnList.writeToString(); + Assert.assertEquals("11:4:4,6,10:2,8", str); + Assert.assertTrue(txnList.isTxnAborted(2L)); + Assert.assertFalse(txnList.isTxnAborted(4L)); + Assert.assertFalse(txnList.isTxnAborted(6L)); + Assert.assertTrue(txnList.isTxnAborted(8L)); + Assert.assertFalse(txnList.isTxnAborted(10L)); + } +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/io/TestDiskRangeList.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/io/TestDiskRangeList.java new file mode 100644 index 0000000000000000000000000000000000000000..a76c7ad7995e919334355f3b4372f057b91e1af8 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/io/TestDiskRangeList.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.io; + +import org.junit.Test; + +public class TestDiskRangeList { + @Test + public void testErrorConditions() throws Exception { + DiskRangeList d510 = new DiskRangeList(0, 10); + d510.insertPartBefore(new DiskRangeList(0, 5)); + DiskRangeList d1015 = d510.insertAfter(new DiskRangeList(10, 15)); + try { + d510.replaceSelfWith(d510); // The arg is self. + fail(); + } catch (AssertionError error) {} + DiskRangeList existing = new DiskRangeList(0, 10); + existing.insertPartBefore(new DiskRangeList(0, 5)); + try { + d510.replaceSelfWith(existing); // The arg is part of another list. + fail(); + } catch (AssertionError error) {} + try { + d510.replaceSelfWith(new DiskRangeList(4, 10)); // Not sequential with previous. + fail(); + } catch (AssertionError error) {} + try { + d510.replaceSelfWith(new DiskRangeList(5, 11)); // Not sequential with next. + fail(); + } catch (AssertionError error) {} + + + try { + d510.insertPartBefore(d510); // The arg is self. + fail(); + } catch (AssertionError error) {} + existing = new DiskRangeList(5, 7); + existing.insertPartBefore(new DiskRangeList(5, 6)); + try { + d510.insertPartBefore(existing); // The arg is part of another list. + fail(); + } catch (AssertionError error) {} + try { + d510.insertPartBefore(new DiskRangeList(3, 4)); // Not a part. + fail(); + } catch (AssertionError error) {} + try { + d510.insertPartBefore(new DiskRangeList(4, 6)); // Not sequential with previous. + fail(); + } catch (AssertionError error) {} + + + try { + d510.insertAfter(d510); // The arg is self. + fail(); + } catch (AssertionError error) {} + existing = new DiskRangeList(15, 20); + existing.insertAfter(new DiskRangeList(20, 25)); + try { + d1015.insertAfter(existing); // The arg is part of another list. + fail(); + } catch (AssertionError error) {} + try { + d1015.insertAfter(new DiskRangeList(14, 20)); // Not sequential. + fail(); + } catch (AssertionError error) {} + d1015.insertAfter(new DiskRangeList(20, 25)); + try { + d1015.insertAfter(new DiskRangeList(15, 21)); // Not sequential with next. + fail(); + } catch (AssertionError error) {} + try { + d1015.insertPartAfter(new DiskRangeList(16, 20)); // Not a part. + fail(); + } catch (AssertionError error) {} + try { + d1015.insertPartAfter(new DiskRangeList(9, 11)); // Not a part. + fail(); + } catch (AssertionError error) {} + + try { + d1015.setEnd(21); // Not sequential with next. + fail(); + } catch (AssertionError error) {} + } + + private void fail() throws Exception { + throw new Exception(); // Don't use Assert.fail, we are catching assertion errors. + } +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/type/HiveDecimalTestBase.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/type/HiveDecimalTestBase.java new file mode 100644 index 0000000000000000000000000000000000000000..9550b948ce3d2d900d143da708f9667cc0bb65e3 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/type/HiveDecimalTestBase.java @@ -0,0 +1,568 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import java.util.Random; +import java.math.BigDecimal; +import java.math.BigInteger; + +import org.apache.hadoop.hive.common.type.RandomTypeUtil; + +public class HiveDecimalTestBase { + + public static int POUND_FACTOR = 1000; + + public static enum BigDecimalFlavor { + NORMAL_RANGE, + FRACTIONS_ONLY, + NEGATIVE_SCALE, + LONG_TAIL + } + + public static enum BigDecimalPairFlavor { + RANDOM, + NEAR, + INVERSE + } + + public BigDecimal randHiveBigDecimal(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + switch (bigDecimalFlavor) { + case NORMAL_RANGE: + return randHiveBigDecimalNormalRange(r, digitAlphabet); + case FRACTIONS_ONLY: + return randHiveBigDecimalFractionsOnly(r, digitAlphabet); + case NEGATIVE_SCALE: + return randHiveBigDecimalNegativeScale(r, digitAlphabet); + case LONG_TAIL: + return randHiveBigDecimalLongTail(r, digitAlphabet); + default: + throw new RuntimeException("Unexpected big decimal flavor " + bigDecimalFlavor); + } + } + + public BigDecimal[] randHiveBigDecimalPair(Random r, String digitAlphabet, + BigDecimalFlavor bigDecimalFlavor, BigDecimalPairFlavor bigDecimalPairFlavor) { + BigDecimal[] pair = new BigDecimal[2]; + BigDecimal bigDecimal1 = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + pair[0] = bigDecimal1; + + BigDecimal bigDecimal2; + switch (bigDecimalPairFlavor) { + case RANDOM: + bigDecimal2 = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + break; + case NEAR: + bigDecimal2 = randHiveBigDecimalNear(r, bigDecimal1); + break; + case INVERSE: + bigDecimal2 = randHiveBigDecimalNear(r, bigDecimal1); + break; + default: + throw new RuntimeException("Unexpected big decimal pair flavor " + bigDecimalPairFlavor); + } + pair[1] = bigDecimal2; + return pair; + } + + public BigDecimal randHiveBigDecimalNormalRange(Random r, String digitAlphabet) { + String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(38)); + BigInteger bigInteger = new BigInteger(digits); + boolean negated = false; + if (r.nextBoolean()) { + bigInteger = bigInteger.negate(); + negated = true; + } + int scale = 0 + r.nextInt(38 + 1); + return new BigDecimal(bigInteger, scale); + } + + public BigDecimal randHiveBigDecimalNegativeScale(Random r, String digitAlphabet) { + String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(38)); + BigInteger bigInteger = new BigInteger(digits); + boolean negated = false; + if (r.nextBoolean()) { + bigInteger = bigInteger.negate(); + negated = true; + } + int scale = 0 + (r.nextBoolean() ? 0 : r.nextInt(38 + 1)); + if (r.nextBoolean()) { + scale = -scale; + } + return new BigDecimal(bigInteger, scale); + } + + public BigDecimal randHiveBigDecimalLongTail(Random r, String digitAlphabet) { + int scale = 0 + r.nextInt(38 + 20); + final int maxDigits = 38 + (scale == 0 ? 0 : 20); + String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(maxDigits)); + BigInteger bigInteger = new BigInteger(digits); + boolean negated = false; + if (r.nextBoolean()) { + bigInteger = bigInteger.negate(); + negated = true; + } + return new BigDecimal(bigInteger, scale); + } + + public BigDecimal randHiveBigDecimalFractionsOnly(Random r, String digitAlphabet) { + int scale = 1 + r.nextInt(38 + 1); + String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(scale)); + BigInteger bigInteger = new BigInteger(digits); + boolean negated = false; + if (r.nextBoolean()) { + bigInteger = bigInteger.negate(); + negated = true; + } + return new BigDecimal(bigInteger, scale); + } + + public BigDecimal randHiveBigDecimalNear(Random r, BigDecimal bigDecimal) { + + int scale = bigDecimal.scale(); + int delta = r.nextInt(10); + if (r.nextBoolean()) { + return bigDecimal.add(new BigDecimal(BigInteger.valueOf(delta), scale)); + } else { + return bigDecimal.subtract(new BigDecimal(BigInteger.valueOf(delta), scale)); + } + } + + public BigDecimal randHiveBigDecimalInverse(Random r, BigDecimal bigDecimal) { + if (bigDecimal.signum() == 0) { + return bigDecimal; + } + return BigDecimal.ONE.divide(bigDecimal); + } + + public BigInteger randHiveBigInteger(Random r, String digitAlphabet) { + String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(38)); + BigInteger bigInteger = new BigInteger(digits); + boolean negated = false; + if (r.nextBoolean()) { + bigInteger = bigInteger.negate(); + negated = true; + } + return bigInteger; + } + + public boolean isTenPowerBug(String string) { + // // System.out.println("TEST_IS_TEN_TO_38_STRING isTenPowerBug " + string); + if (string.charAt(0) == '-') { + string = string.substring(1); + } + int index = string.indexOf('.'); + if (index != -1) { + if (index == 0) { + string = string.substring(1); + } else { + string = string.substring(0, index) + string.substring(index + 1); + } + } + // // System.out.println("TEST_IS_TEN_TO_38_STRING isTenPowerBug " + string); + return string.equals("100000000000000000000000000000000000000"); + } + + //------------------------------------------------------------------------------------------------ + + public static String[] specialDecimalStrings = new String[] { + "0", + "1", + "-1", + "10", + "-10", + "100", + "-100", + "127", // Byte.MAX_VALUE + "127.1", + "127.0008", + "127.49", + "127.5", + "127.9999999999999999999", + "-127", + "-127.1", + "-127.0008", + "-127.49", + "-127.5", + "-127.999999", + "128", + "128.1", + "128.0008", + "128.49", + "128.5", + "128.9999999999999999999", + "-128", // Byte.MIN_VALUE + "-128.1", + "-128.0008", + "-128.49", + "-128.5", + "-128.999", + "129", + "129.1", + "-129", + "-129.1", + "1000", + "-1000", + "10000", + "-10000", + "32767", // Short.MAX_VALUE + "32767.1", + "32767.0008", + "32767.49", + "32767.5", + "32767.99999999999", + "-32767", + "-32767.1", + "-32767.0008", + "-32767.49", + "-32767.5", + "-32767.9", + "32768", + "32768.1", + "32768.0008", + "32768.49", + "32768.5", + "32768.9999999999", + "-32768", // Short.MIN_VALUE + "-32768.1", + "-32768.0008", + "-32768.49", + "-32768.5", + "-32768.9999999", + "32769", + "32769.1", + "-32769", + "-32769.1", + "100000", + "-100000", + "1000000", + "-1000000", + "10000000", + "-10000000", + "100000000", + "99999999", // 10^8 - 1 + "-99999999", + "-100000000", + "1000000000", + "-1000000000", + "2147483647", // Integer.MAX_VALUE + "2147483647.1", + "2147483647.0008", + "2147483647.49", + "2147483647.5", + "2147483647.9999999999", + "-2147483647", + "-2147483647.1", + "-2147483647.0008", + "-2147483647.49", + "-2147483647.5", + "-2147483647.9999999999999999999", + "2147483648", + "2147483648.1", + "2147483648.0008", + "2147483648.49", + "2147483648.5", + "2147483648.9", + "-2147483648", // Integer.MIN_VALUE + "-2147483648.1", + "-2147483648.0008", + "-2147483648.49", + "-2147483648.5", + "-2147483648.999", + "2147483649", + "2147483649.1", + "-2147483649", + "-2147483649.1", + "10000000000", + "-10000000000", + "100000000000", + "-100000000000", + "1000000000000", + "-1000000000000", + "10000000000000", + "-10000000000000", + "100000000000000", + "-100000000000000", + "999999999999999", + "-999999999999999", + "1000000000000000", // 10^15 + "-1000000000000000", + "9999999999999999", // 10^16 - 1 + "-9999999999999999", + "10000000000000000", // 10^16 + "-10000000000000000", + "99999999999999999", // 10^17 - 1 + "-99999999999999999", + "100000000000000000", + "-100000000000000000", + "999999999999999999", // 10^18 - 1 + "-999999999999999999", + "123456789012345678", + "-123456789012345678", + "1000000000000000000", + "-1000000000000000000", + "9223372036854775807", // Long.MAX_VALUE + "9223372036854775807.1", + "9223372036854775807.0008", + "9223372036854775807.49", + "9223372036854775807.5", + "9223372036854775807.9", + "-9223372036854775807", + "-9223372036854775807.1", + "-9223372036854775807.0008", + "-9223372036854775807.49", + "-9223372036854775807.5", + "-9223372036854775807.9999999999999999999", + "-9223372036854775808", + "-9223372036854775808.1", + "9223372036854775808", + "9223372036854775808.1", + "9223372036854775808.0008", + "9223372036854775808.49", + "9223372036854775808.5", + "9223372036854775808.9", + "9223372036854775809", + "9223372036854775809.1", + "-9223372036854775808", // Long.MIN_VALUE + "-9223372036854775808.1", + "-9223372036854775808.0008", + "-9223372036854775808.49", + "-9223372036854775808.5", + "-9223372036854775808.9999999", + "9223372036854775809", + "9223372036854775809.1", + "-9223372036854775809", + "-9223372036854775809.1", + "10000000000000000000000000000000", // 10^31 + "-10000000000000000000000000000000", + "99999999999999999999999999999999", // 10^32 - 1 + "-99999999999999999999999999999999", + "100000000000000000000000000000000", // 10^32 + "-100000000000000000000000000000000", + "10000000000000000000000000000000000000", // 10^37 + "-10000000000000000000000000000000000000", + "99999999999999999999999999999999999999", // 10^38 - 1 + "-99999999999999999999999999999999999999", + "100000000000000000000000000000000000000", // 10^38 + "-100000000000000000000000000000000000000", + "1000000000000000000000000000000000000000", // 10^39 + "-1000000000000000000000000000000000000000", + + "18446744073709551616", // Unsigned 64 max. + "-18446744073709551616", + "340282366920938463463374607431768211455", // 2^128 - 1 + "-340282366920938463463374607431768211455", + + "0.999999999999999", + "-0.999999999999999", + "0.0000000000000001", // 10^-15 + "-0.0000000000000001", + "0.9999999999999999", + "-0.9999999999999999", + "0.00000000000000001", // 10^-16 + "-0.00000000000000001", + "0.99999999999999999", + "-0.99999999999999999", + "0.999999999999999999", // 10^-18 + "-0.999999999999999999", + "0.00000000000000000000000000000001", // 10^-31 + "-0.00000000000000000000000000000001", + "0.99999999999999999999999999999999", // 10^-32 + 1 + "-0.99999999999999999999999999999999", + "0.000000000000000000000000000000001", // 10^-32 + "-0.000000000000000000000000000000001", + "0.00000000000000000000000000000000000001", // 10^-37 + "-0.00000000000000000000000000000000000001", + "0.99999999999999999999999999999999999999", // 10^-38 + 1 + "-0.99999999999999999999999999999999999999", + "0.000000000000000000000000000000000000001", // 10^-38 + "-0.000000000000000000000000000000000000001", + "0.0000000000000000000000000000000000000001", // 10^-39 + "-0.0000000000000000000000000000000000000001", + "0.0000000000000000000000000000000000000005", // 10^-39 (rounds) + "-0.0000000000000000000000000000000000000005", + "0.340282366920938463463374607431768211455", // (2^128 - 1) * 10^-39 + "-0.340282366920938463463374607431768211455", + "0.000000000000000000000000000000000000001", // 10^-38 + "-0.000000000000000000000000000000000000001", + "0.000000000000000000000000000000000000005", // 10^-38 + "-0.000000000000000000000000000000000000005", + + "234.79", + "342348.343", + "12.25", + "-12.25", + "72057594037927935", // 2^56 - 1 + "-72057594037927935", + "72057594037927936", // 2^56 + "-72057594037927936", + "5192296858534827628530496329220095", // 2^56 * 2^56 - 1 + "-5192296858534827628530496329220095", + "5192296858534827628530496329220096", // 2^56 * 2^56 + "-5192296858534827628530496329220096", + + "54216721532321902598.70", + "-906.62545207002374150309544832320", + "-0.0709351061072", + "1460849063411925.53", + "8.809130E-33", + "-4.0786300706013636202E-20", + "-3.8823936518E-1", + "-3.8823936518E-28", + "-3.8823936518E-29", + "598575157855521918987423259.94094", + "299999448432.001342152474197", + "1786135888657847525803324040144343378.09799306448796128931113691624", // More than 38 digits. + "-1786135888657847525803324040144343378.09799306448796128931113691624", + "57847525803324040144343378.09799306448796128931113691624", + "0.999999999999999999990000", + "005.34000", + "1E-90", + + "0.4", + "-0.4", + "0.5", + "-0.5", + "0.6", + "-0.6", + "1.4", + "-1.4", + "1.5", + "-1.5", + "1.6", + "-1.6", + "2.4", + "-2.4", + "2.49", + "-2.49", + "2.5", + "-2.5", + "2.51", + "-2.51", + "-2.5", + "2.6", + "-2.6", + "3.00001415926", + "0.00", + "-12.25", + "234.79" + }; + + public static BigDecimal[] specialBigDecimals = stringArrayToBigDecimals(specialDecimalStrings); + + // decimal_1_1.txt + public static String[] decimal_1_1_txt = { + "0.0", + "0.0000", + ".0", + "0.1", + "0.15", + "0.9", + "0.94", + "0.99", + "0.345", + "1.0", + "1", + "0", + "00", + "22", + "1E-9", + "-0.0", + "-0.0000", + "-.0", + "-0.1", + "-0.15", + "-0.9", + "-0.94", + "-0.99", + "-0.345", + "-1.0", + "-1", + "-0", + "-00", + "-22", + "-1E-9" + }; + + // kv7.txt KEYS + public static String[] kv7_txt_keys = { + "-4400", + "1E+99", + "1E-99", + "0", + "100", + "10", + "1", + "0.1", + "0.01", + "200", + "20", + "2", + "0", + "0.2", + "0.02", + "0.3", + "0.33", + "0.333", + "-0.3", + "-0.33", + "-0.333", + "1.0", + "2", + "3.14", + "-1.12", + "-1.12", + "-1.122", + "1.12", + "1.122", + "124.00", + "125.2", + "-1255.49", + "3.14", + "3.14", + "3.140", + "0.9999999999999999999999999", + "-1234567890.1234567890", + "1234567890.1234567800" + }; + + public static String standardAlphabet = "0123456789"; + + public static String[] sparseAlphabets = new String[] { + + "0000000000000000000000000000000000000003", + "0000000000000000000000000000000000000009", + "0000000000000000000000000000000000000001", + "0000000000000000000003", + "0000000000000000000009", + "0000000000000000000001", + "0000000000091", + "000000000005", + "9", + "5555555555999999999000000000000001111111", + "24680", + "1" + }; + + public static BigDecimal[] stringArrayToBigDecimals(String[] strings) { + BigDecimal[] result = new BigDecimal[strings.length]; + for (int i = 0; i < strings.length; i++) { + result[i] = new BigDecimal(strings[i]); + } + return result; + } +} \ No newline at end of file diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java new file mode 100644 index 0000000000000000000000000000000000000000..d11f41c65f56487e1e267130a027d196d332908a --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java @@ -0,0 +1,3477 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import java.sql.Timestamp; +import java.util.Random; +import java.util.Arrays; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; + +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritableV1; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.common.type.RandomTypeUtil; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; +import org.apache.hadoop.hive.ql.util.TimestampUtils; + +import org.junit.*; + +import static org.junit.Assert.*; + +public class TestHiveDecimal extends HiveDecimalTestBase { + + @Test + public void testInvalidStringInput() { + + HiveDecimalV1 oldDec; + HiveDecimalV1 resultOldDec; + HiveDecimal dec; + HiveDecimal resultDec; + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("-"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("-"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("+"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("+"); + Assert.assertTrue(dec == null); + + // Naked dot. + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("."); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("."); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("-."); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("-."); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("+."); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("+."); + Assert.assertTrue(dec == null); + + // Naked E/e. + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("E"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("E"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(".E"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create(".E"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("-E"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("-E"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("+E"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("+E"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("e"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("e"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(".e"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create(".e"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("-e"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("-e"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("+e"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("+e"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("error"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("error"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("0x0"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("0x0"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("0e"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("0e"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("7e"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("7e"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("233e-"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("233e-"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("32e+"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("32e+"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(".0e"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create(".0e"); + Assert.assertTrue(dec == null); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(".4e"); + Assert.assertTrue(oldDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create(".4e"); + Assert.assertTrue(dec == null); + } + + @Test + public void testVariousCases() { + + HiveDecimalV1 oldDec; + HiveDecimalV1 resultOldDec; + HiveDecimal dec; + HiveDecimal resultDec; + + BigDecimal bigDecimal = new BigDecimal("-99999999999999999999999999999999999999.99999999999999999"); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(bigDecimal); + Assert.assertEquals("-100000000000000000000000000000000000000", oldDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(bigDecimal); + Assert.assertTrue(dec == null); + + // One less integer digit... + bigDecimal = new BigDecimal("-9999999999999999999999999999999999999.99999999999999999"); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(bigDecimal); + Assert.assertEquals("-10000000000000000000000000000000000000", oldDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(bigDecimal); + Assert.assertEquals("-10000000000000000000000000000000000000", dec.toString()); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("101"); + resultOldDec = HiveDecimalV1.enforcePrecisionScale(oldDec, 10, 0); + Assert.assertEquals("101", resultOldDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create("101"); + resultDec = HiveDecimal.enforcePrecisionScale(dec, 10, 0); + Assert.assertEquals("101", resultDec.toString()); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("1"); + resultOldDec = oldDec.scaleByPowerOfTen(-99); + Assert.assertEquals("0", resultOldDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create("1"); + resultDec = dec.scaleByPowerOfTen(-99); + Assert.assertEquals("0", resultDec.toString()); + } + + @Test + public void testCreateFromBigIntegerRounding() { + + BigInteger bigInt; + HiveDecimalV1 oldDec; + HiveDecimal dec; + + // 1786135888657847525803324040144343378.09799306448796128931113691624 + bigInt = new BigInteger( + "178613588865784752580332404014434337809799306448796128931113691624"); + Assert.assertEquals("178613588865784752580332404014434337809799306448796128931113691624", bigInt.toString()); + // 12345678901234567890123456789012345678 + // 1 2 3 + // 12345678901234567890123456789 + dec = HiveDecimal.create(bigInt, 29); + Assert.assertEquals("1786135888657847525803324040144343378.1", dec.toString()); + + // 8.090000000000000000000000000000000000000123456 + bigInt = new BigInteger( + "8090000000000000000000000000000000000000123456"); + // 123456789012345678901234567890123456789012345 + // 1 2 3 4 + Assert.assertEquals("8090000000000000000000000000000000000000123456", bigInt.toString()); + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(bigInt, 45); + Assert.assertEquals("8.09", oldDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(bigInt, 45); + Assert.assertEquals("8.09", dec.toString()); + + // 99999999.99999999999999999999999999999949999 + // MAX_DECIMAL 9's WITH NO ROUND (longer than 38 digits) + bigInt = new BigInteger( + "9999999999999999999999999999999999999949999"); + // 12345678901234567890123456789012345678 + // 1 2 3 + // 99999999.99999999999999999999999999999949999 + Assert.assertEquals("9999999999999999999999999999999999999949999", bigInt.toString()); + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(bigInt, 35); + Assert.assertEquals("99999999.999999999999999999999999999999", oldDec.toString()); + //--------------------------------------------------- + // Without the round, this conversion fails. + dec = HiveDecimal.create(bigInt, 35); + Assert.assertEquals("99999999.999999999999999999999999999999", dec.toString()); + + // MAX_DECIMAL 9's WITH ROUND. + bigInt = new BigInteger( + "9999999999999999999999999999999999999979999"); + // 12346678.901234667890123466789012346678 + // 1 2 3 + Assert.assertEquals("9999999999999999999999999999999999999979999", bigInt.toString()); + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(bigInt, 35); + Assert.assertEquals("100000000", oldDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(bigInt, 35); + Assert.assertEquals("100000000", dec.toString()); + } + + @Test + public void testCreateFromBigDecimal() { + + BigDecimal bigDec; + HiveDecimalV1 oldDec; + HiveDecimal dec; + + bigDec = new BigDecimal("0"); + Assert.assertEquals("0", bigDec.toString()); + dec = HiveDecimal.create(bigDec); + Assert.assertEquals("0", dec.toString()); + + bigDec = new BigDecimal("1"); + Assert.assertEquals("1", bigDec.toString()); + dec = HiveDecimal.create(bigDec); + Assert.assertEquals("1", dec.toString()); + + bigDec = new BigDecimal("0.999"); + Assert.assertEquals("0.999", bigDec.toString()); + dec = HiveDecimal.create(bigDec); + Assert.assertEquals("0.999", dec.toString()); + + // HiveDecimal suppresses trailing zeroes. + bigDec = new BigDecimal("0.9990"); + Assert.assertEquals("0.9990", bigDec.toString()); + dec = HiveDecimal.create(bigDec); + Assert.assertEquals("0.999", dec.toString()); + } + + @Test + public void testCreateFromBigDecimalRounding() { + + BigDecimal bigDec; + HiveDecimalV1 oldDec; + HiveDecimal dec; + + bigDec = new BigDecimal( + "1786135888657847525803324040144343378.09799306448796128931113691624"); + Assert.assertEquals("1786135888657847525803324040144343378.09799306448796128931113691624", bigDec.toString()); + // 1234567890123456789012345678901234567.8 + // 1 2 3 + // Without the round, this conversion fails. + dec = HiveDecimal.create(bigDec, false); + Assert.assertTrue(dec == null); + dec = HiveDecimal.create(bigDec, true); + Assert.assertEquals("1786135888657847525803324040144343378.1", dec.toString()); + + bigDec = new BigDecimal( + "8.090000000000000000000000000000000000000123456"); + // 1.23456789012345678901234567890123456789012345 + // 1 2 3 4 + Assert.assertEquals("8.090000000000000000000000000000000000000123456", bigDec.toString()); + //--------------------------------------------------- + HiveDecimalV1 oldDec4 = HiveDecimalV1.create(bigDec, false); + Assert.assertTrue(oldDec4 == null); + oldDec4 = HiveDecimalV1.create(bigDec, true); + Assert.assertEquals("8.09", oldDec4.toString()); + //--------------------------------------------------- + // Without the round, this conversion fails. + dec = HiveDecimal.create(bigDec, false); + Assert.assertTrue(dec == null); + dec = HiveDecimal.create(bigDec, true); + Assert.assertEquals("8.09", dec.toString()); + + // MAX_DECIMAL 9's WITH NO ROUND (longer than 38 digits) + bigDec = new BigDecimal( + "99999999.99999999999999999999999999999949999"); + // 12345678.901234567890123456789012345678 + // 1 2 3 + Assert.assertEquals("99999999.99999999999999999999999999999949999", bigDec.toString()); + //--------------------------------------------------- + HiveDecimalV1 oldDec5 = HiveDecimalV1.create(bigDec, false); + Assert.assertTrue(oldDec5 == null); + oldDec5 = HiveDecimalV1.create(bigDec, true); + Assert.assertEquals("99999999.999999999999999999999999999999", oldDec5.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(bigDec, false); + Assert.assertTrue(dec == null); + dec = HiveDecimal.create(bigDec, true); + Assert.assertEquals("99999999.999999999999999999999999999999", dec.toString()); + + // MAX_DECIMAL 9's WITH ROUND. + bigDec = new BigDecimal( + "99999999.99999999999999999999999999999979999"); + // 12346678.901234667890123466789012346678 + // 1 2 3 + Assert.assertEquals("99999999.99999999999999999999999999999979999", bigDec.toString()); + //--------------------------------------------------- + HiveDecimalV1 oldDec6 = HiveDecimalV1.create(bigDec, false); + Assert.assertTrue(oldDec6 == null); + oldDec6 = HiveDecimalV1.create(bigDec, true); + Assert.assertEquals("100000000", oldDec6.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(bigDec, false); + Assert.assertTrue(dec == null); + dec = HiveDecimal.create(bigDec, true); + Assert.assertEquals("100000000", dec.toString()); + } + + + @Test + public void testPrecisionScaleEnforcement() { + + HiveDecimalV1 oldDec; + HiveDecimalV1 oldResultDec; + + HiveDecimal dec; + HiveDecimal resultDec; + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("0.02538461538461538461538461538461538462"); + Assert.assertEquals("0.02538461538461538461538", + HiveDecimalV1.enforcePrecisionScale(oldDec, 38, 23).toString()); + //--------------------------------------------------- + dec = HiveDecimal.create("0.02538461538461538461538461538461538462"); + Assert.assertEquals("0.02538461538461538461538", + HiveDecimal.enforcePrecisionScale(dec, 38, 23).toString()); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("005.34000"); + Assert.assertEquals(oldDec.precision(), 3); // 1 integer digit; 2 fraction digits. + Assert.assertEquals(oldDec.scale(), 2); // Trailing zeroes are suppressed. + //--------------------------------------------------- + dec = HiveDecimal.create("005.34000"); + Assert.assertEquals(dec.precision(), 3); // 1 integer digit; 2 fraction digits. + Assert.assertEquals(dec.scale(), 2); // Trailing zeroes are suppressed. + + dec = HiveDecimal.create("178613588865784752580332404014434337809799306448796128931113691624"); + Assert.assertNull(dec); + + // Rounding numbers that increase int digits + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("9.5"); + Assert.assertEquals("10", + HiveDecimalV1.enforcePrecisionScale(oldDec, 2, 0).toString()); + Assert.assertNull( + HiveDecimalV1.enforcePrecisionScale(oldDec, 1, 0)); + oldDec = HiveDecimalV1.create("9.4"); + Assert.assertEquals("9", + HiveDecimalV1.enforcePrecisionScale(oldDec, 1, 0).toString()); + //--------------------------------------------------- + dec = HiveDecimal.create("9.5"); + Assert.assertEquals("10", + HiveDecimal.enforcePrecisionScale(dec, 2, 0).toString()); + Assert.assertNull( + HiveDecimal.enforcePrecisionScale(dec, 1, 0)); + dec = HiveDecimal.create("9.4"); + Assert.assertEquals("9", + HiveDecimal.enforcePrecisionScale(dec, 1, 0).toString()); + } + + @Test + public void testPrecisionScaleEnforcementEdgeCond() { + + // Since HiveDecimal now uses FastHiveDecimal which stores 16 decimal digits per long, + // lets test edge conditions here. + + HiveDecimal fifteenFractionalNinesDec = HiveDecimal.create("0.999999999999999"); + Assert.assertNotNull(fifteenFractionalNinesDec); + Assert.assertEquals("0.999999999999999", + HiveDecimal.enforcePrecisionScale(fifteenFractionalNinesDec, 15, 15).toString()); + + HiveDecimal sixteenFractionalNines = HiveDecimal.create("0.9999999999999999"); + Assert.assertNotNull(sixteenFractionalNines); + Assert.assertEquals("0.9999999999999999", + HiveDecimal.enforcePrecisionScale(sixteenFractionalNines, 16, 16).toString()); + + HiveDecimal seventeenFractionalNines = HiveDecimal.create("0.99999999999999999"); + Assert.assertNotNull(seventeenFractionalNines); + Assert.assertEquals("0.99999999999999999", + HiveDecimal.enforcePrecisionScale(seventeenFractionalNines, 17, 17).toString()); + + } + + @Test + public void testTrailingZeroRemovalAfterEnforcement() { + String decStr = "8.090000000000000000000000000000000000000123456"; + // 123456789012345678901234567890123456789012345 + // 1 2 3 4 + HiveDecimal dec = HiveDecimal.create(decStr); + Assert.assertEquals("8.09", dec.toString()); + } + + @Test + public void testMultiply() { + + // This multiply produces more than 38 digits --> overflow. + //--------------------------------------------------- + HiveDecimalV1 oldDec1 = HiveDecimalV1.create("0.00001786135888657847525803"); + HiveDecimalV1 oldDec2 = HiveDecimalV1.create("3.0000123456789"); + HiveDecimalV1 oldResult = oldDec1.multiply(oldDec2); + Assert.assertTrue(oldResult == null); + //--------------------------------------------------- + HiveDecimal dec1 = HiveDecimal.create("0.00001786135888657847525803"); + HiveDecimal dec2 = HiveDecimal.create("3.0000123456789"); + HiveDecimal result = dec1.multiply(dec2); + Assert.assertTrue(result == null); + + dec1 = HiveDecimal.create("178613588865784752580323232232323444.4"); + dec2 = HiveDecimal.create("178613588865784752580302323232.3"); + Assert.assertNull(dec1.multiply(dec2)); // i.e. Overflow. + + dec1 = HiveDecimal.create("47.324"); + dec2 = HiveDecimal.create("9232.309"); + Assert.assertEquals("436909.791116", dec1.multiply(dec2).toString()); + + dec1 = HiveDecimal.create("3.140"); + dec2 = HiveDecimal.create("1.00"); + Assert.assertEquals("3.14", dec1.multiply(dec2).toString()); + + dec1 = HiveDecimal.create("43.010"); + dec2 = HiveDecimal.create("2"); + Assert.assertEquals("86.02", dec1.multiply(dec2).toString()); + } + + @Test + public void testMultiply2() { + // 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD + HiveDecimal dec1 = HiveDecimal.create("0.09765625"); + HiveDecimal dec2 = HiveDecimal.create("0.09765625"); + HiveDecimal dec3 = HiveDecimal.create("0.0125"); + HiveDecimal dec4 = HiveDecimal.create("578992"); + HiveDecimal result1 = dec1.multiply(dec2); + Assert.assertNotNull(result1); + HiveDecimal result2 = result1.multiply(dec3); + Assert.assertNotNull(result2); + HiveDecimal result = result2.multiply(dec4); + Assert.assertNotNull(result); + Assert.assertEquals("69.0212249755859375", result.toString()); + } + + @Test + public void testPow() { + + HiveDecimal dec; + + dec = HiveDecimal.create("3.00001415926"); + HiveDecimal decPow2 = dec.pow(2); + HiveDecimal decMultiplyTwice = dec.multiply(dec); + Assert.assertEquals(decPow2, decMultiplyTwice); + + dec = HiveDecimal.create("0.000017861358882"); + dec = dec.pow(3); + Assert.assertNull(dec); + + dec = HiveDecimal.create("3.140"); + Assert.assertEquals("9.8596", dec.pow(2).toString()); + } + + @Test + public void testScaleByPowerOfTen() { + + HiveDecimalV1 oldDec; + HiveDecimal dec; + HiveDecimalV1 oldResultDec; + HiveDecimal resultDec; + + //********************************************************************************************** + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create( + "1"); + Assert.assertEquals(0, oldDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(2); + Assert.assertEquals( + "100", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create( + "1"); + Assert.assertEquals(0, dec.scale()); + // resultDec = dec.scaleByPowerOfTen(2); + // Assert.assertEquals( + // "100", resultDec.toString()); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, oldDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(2); + Assert.assertEquals( + "0.000000000000000000000000000000000001", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, dec.scale()); + resultDec = dec.scaleByPowerOfTen(2); + Assert.assertEquals( + "0.000000000000000000000000000000000001", resultDec.toString()); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, oldDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(38); + Assert.assertEquals( + "1", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, dec.scale()); + resultDec = dec.scaleByPowerOfTen(38); + Assert.assertEquals( + "1", resultDec.toString()); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, oldDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(2 * 38 - 1); + Assert.assertEquals( + "10000000000000000000000000000000000000", oldResultDec.toString()); + Assert.assertEquals(0, oldResultDec.scale()); + //--------------------------------------------------- + dec = HiveDecimal.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, dec.scale()); + resultDec = dec.scaleByPowerOfTen(2 * 38 - 1); + Assert.assertEquals( + "10000000000000000000000000000000000000", resultDec.toString()); + Assert.assertEquals(0, resultDec.scale()); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, oldDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(2 * 38); + Assert.assertTrue(oldResultDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, dec.scale()); + resultDec = dec.scaleByPowerOfTen(2 * 38); + Assert.assertTrue(resultDec == null); + + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create( + "0.00000000000000000000000000000000000022"); + Assert.assertEquals(38, oldDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(38); + Assert.assertEquals( + "22", oldResultDec.toString()); + Assert.assertEquals(0, oldResultDec.scale()); + //--------------------------------------------------- + dec = HiveDecimal.create( + "0.00000000000000000000000000000000000022"); + Assert.assertEquals(38, dec.scale()); + resultDec = dec.scaleByPowerOfTen(38); + Assert.assertEquals( + "22", resultDec.toString()); + Assert.assertEquals(0, resultDec.scale()); + + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("3.00001415926"); + Assert.assertEquals(11, oldDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(2); + Assert.assertEquals("300.001415926", oldResultDec.toString()); + Assert.assertEquals(9, oldResultDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(5); + Assert.assertEquals("300001.415926", oldResultDec.toString()); + Assert.assertEquals(6, oldResultDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(18); + Assert.assertEquals("3000014159260000000", oldResultDec.toString()); + Assert.assertEquals(0, oldResultDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(35); + Assert.assertEquals("300001415926000000000000000000000000", oldResultDec.toString()); + Assert.assertEquals(0, oldResultDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(37); + Assert.assertEquals("30000141592600000000000000000000000000", oldResultDec.toString()); + Assert.assertEquals(0, oldResultDec.scale()); + //--------------------------------------------------- + dec = HiveDecimal.create("3.00001415926"); + Assert.assertEquals(11, dec.scale()); + Assert.assertEquals(1, dec.integerDigitCount()); + resultDec = dec.scaleByPowerOfTen(2); + Assert.assertEquals("300.001415926", resultDec.toString()); + Assert.assertEquals(9, resultDec.scale()); + Assert.assertEquals(3, resultDec.integerDigitCount()); + resultDec = dec.scaleByPowerOfTen(5); + Assert.assertEquals("300001.415926", resultDec.toString()); + Assert.assertEquals(6, resultDec.scale()); + Assert.assertEquals(6, resultDec.integerDigitCount()); + resultDec = dec.scaleByPowerOfTen(18); + Assert.assertEquals("3000014159260000000", resultDec.toString()); + Assert.assertEquals(0, resultDec.scale()); + Assert.assertEquals(19, resultDec.integerDigitCount()); + resultDec = dec.scaleByPowerOfTen(35); + Assert.assertEquals("300001415926000000000000000000000000", resultDec.toString()); + Assert.assertEquals(0, resultDec.scale()); + Assert.assertEquals(36, resultDec.integerDigitCount()); + resultDec = dec.scaleByPowerOfTen(37); + Assert.assertEquals("30000141592600000000000000000000000000", resultDec.toString()); + Assert.assertEquals(0, resultDec.scale()); + Assert.assertEquals(38, resultDec.integerDigitCount()); + } + + @Test + public void testSingleWordDivision() { + + HiveDecimalV1 oldDec1; + HiveDecimalV1 oldDec2; + HiveDecimalV1 oldResultDec; + + HiveDecimal dec1; + HiveDecimal dec2; + HiveDecimal resultDec; + + //--------------------------------------------------- + oldDec1 = HiveDecimalV1.create("839293"); + oldDec2 = HiveDecimalV1.create("8"); + oldResultDec = oldDec1.divide(oldDec2); + Assert.assertEquals("104911.625", oldResultDec.toString()); + //--------------------------------------------------- + dec1 = HiveDecimal.create("839293"); + dec2 = HiveDecimal.create("8"); + resultDec = dec1.divide(dec2); + Assert.assertEquals("104911.625", resultDec.toString()); // UNDONE + + //--------------------------------------------------- + oldDec1 = HiveDecimalV1.create("1"); + oldDec2 = HiveDecimalV1.create("3"); + oldResultDec = oldDec1.divide(oldDec2); + Assert.assertEquals("0.33333333333333333333333333333333333333", oldResultDec.toString()); + //--------------------------------------------------- + dec1 = HiveDecimal.create("1"); + dec2 = HiveDecimal.create("3"); + resultDec = dec1.divide(dec2); + Assert.assertEquals("0.33333333333333333333333333333333333333", resultDec.toString()); // UNDONE + + //--------------------------------------------------- + oldDec1 = HiveDecimalV1.create("1"); + oldDec2 = HiveDecimalV1.create("9"); + oldResultDec = oldDec1.divide(oldDec2); + Assert.assertEquals("0.11111111111111111111111111111111111111", oldResultDec.toString()); + //--------------------------------------------------- + dec1 = HiveDecimal.create("1"); + dec2 = HiveDecimal.create("9"); + resultDec = dec1.divide(dec2); + Assert.assertEquals("0.11111111111111111111111111111111111111", resultDec.toString()); // UNDONE + + //--------------------------------------------------- + oldDec1 = HiveDecimalV1.create("22"); + oldDec2 = HiveDecimalV1.create("7"); + oldResultDec = oldDec1.divide(oldDec2); + Assert.assertEquals("3.1428571428571428571428571428571428571", oldResultDec.toString()); + //--------------------------------------------------- + dec1 = HiveDecimal.create("22"); + dec2 = HiveDecimal.create("7"); + resultDec = dec1.divide(dec2); + Assert.assertEquals("3.1428571428571428571428571428571428571", resultDec.toString()); // UNDONE + + //--------------------------------------------------- + oldDec1 = HiveDecimalV1.create("1"); + oldDec2 = HiveDecimalV1.create("81"); + oldResultDec = oldDec1.divide(oldDec2); + Assert.assertEquals("0.01234567901234567901234567901234567901", oldResultDec.toString()); + //--------------------------------------------------- + dec1 = HiveDecimal.create("1"); + dec2 = HiveDecimal.create("81"); + resultDec = dec1.divide(dec2); + Assert.assertEquals("0.01234567901234567901234567901234567901", resultDec.toString()); // UNDONE + + //--------------------------------------------------- + oldDec1 = HiveDecimalV1.create("425"); + oldDec2 = HiveDecimalV1.create("1000000000000000"); + oldResultDec = oldDec1.divide(oldDec2); + Assert.assertEquals("0.000000000000425", oldResultDec.toString()); + //--------------------------------------------------- + dec1 = HiveDecimal.create("425"); + dec2 = HiveDecimal.create("1000000000000000"); + resultDec = dec1.divide(dec2); + Assert.assertEquals("0.000000000000425", resultDec.toString()); // UNDONE + + //--------------------------------------------------- + oldDec1 = HiveDecimalV1.create("0.000000000088"); + oldDec2 = HiveDecimalV1.create("1000000000000000"); + oldResultDec = oldDec1.divide(oldDec2); + Assert.assertEquals("0.000000000000000000000000088", oldResultDec.toString()); + Assert.assertEquals(27, oldResultDec.scale()); + //--------------------------------------------------- + dec1 = HiveDecimal.create("0.000000000088"); + dec2 = HiveDecimal.create("1000000000000000"); + resultDec = dec1.divide(dec2); + Assert.assertEquals("0.000000000000000000000000088", resultDec.toString()); // UNDONE + Assert.assertEquals(27, resultDec.scale()); + } + + @Test + public void testDivide() { + HiveDecimal dec1 = HiveDecimal.create("3.14"); + HiveDecimal dec2 = HiveDecimal.create("3"); + Assert.assertNotNull(dec1.divide(dec2)); + + dec1 = HiveDecimal.create("15"); + dec2 = HiveDecimal.create("5"); + Assert.assertEquals("3", dec1.divide(dec2).toString()); + + dec1 = HiveDecimal.create("3.140"); + dec2 = HiveDecimal.create("1.00"); + Assert.assertEquals("3.14", dec1.divide(dec2).toString()); + } + + @Test + public void testPlus() { + + HiveDecimalV1 oldDec; + HiveDecimalV1 oldDec2; + HiveDecimalV1 oldResultDec; + + HiveDecimal dec; + HiveDecimal dec1; + HiveDecimal dec2; + HiveDecimal resultDec; + + String decStr; + String decStr2; + + dec1 = HiveDecimal.create("3.140"); + dec1.validate(); + dec2 = HiveDecimal.create("1.00"); + dec2.validate(); + resultDec = dec1.add(dec2); + resultDec.validate(); + Assert.assertEquals("4.14", resultDec.toString()); + + decStr = "3.140"; + decStr2 = "1.00"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("4.14", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("4.14", resultDec.toString()); + Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "3.140"; + decStr2 = "1.00000008733"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("4.14000008733", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("4.14000008733", resultDec.toString()); + Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "3.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("4.14000000000000000008733", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("4.14000000000000000008733", resultDec.toString()); + Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "30000000000.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("30000000001.14000000000000000008733", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("30000000001.14000000000000000008733", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "300000000000000.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("300000000000001.14000000000000000008733", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("300000000000001.14000000000000000008733", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + // Edge case? + decStr = "3000000000000000.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("3000000000000001.1400000000000000000873", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("3000000000000001.1400000000000000000873", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "300000000000000000000000.14"; + decStr2 = "0.0000055555555550008733"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("300000000000000000000000.14000555555556", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("300000000000000000000000.14000555555556", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "300000000000000000000000.14"; + decStr2 = "0.000005555555555000873355"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("300000000000000000000000.14000555555556", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("300000000000000000000000.14000555555556", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + + + // Example from HiveDecimal.add header comments. + decStr = "598575157855521918987423259.94094"; + decStr2 = "0.0000000000006711991169422033"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("598575157855521918987423259.94094", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("598575157855521918987423259.94094", resultDec.toString()); + Assert.assertEquals(27, resultDec.integerDigitCount()); + + decStr = "598575157855521918987423259.94094"; + decStr2 = "0.5555555555556711991169422033"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("598575157855521918987423260.49649555556", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("598575157855521918987423260.49649555556", resultDec.toString()); + Assert.assertEquals(27, resultDec.integerDigitCount()); + + decStr = "199999999.99995"; + decStr2 = "100000000.00005"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("300000000", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("300000000", resultDec.toString()); + Assert.assertEquals(9, resultDec.integerDigitCount()); + + dec1 = HiveDecimal.create("99999999999999999999999999999999999999"); + dec1.validate(); + Assert.assertEquals(38, dec1.integerDigitCount()); + dec2 = HiveDecimal.create("1"); + dec2.validate(); + Assert.assertNull(dec1.add(dec2)); + } + + @Test + public void testAdd() { + + HiveDecimalV1 oldDec; + HiveDecimalV1 oldDec2; + HiveDecimalV1 oldResultDec; + + HiveDecimal dec; + HiveDecimal dec2; + HiveDecimal resultDec; + + // Use the example from HIVE-13423 where the integer digits of the result exceed the + // enforced precision/scale. + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("98765432109876543210.12345"); + oldResultDec = oldDec.add(oldDec); + Assert.assertEquals("197530864219753086420.2469", oldResultDec.toString()); + oldResultDec = HiveDecimalV1.enforcePrecisionScale(oldResultDec, 38, 18); + Assert.assertTrue(oldResultDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("98765432109876543210.12345"); + assertTrue(dec != null); + dec.validate(); + resultDec = dec.add(dec); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("197530864219753086420.2469", resultDec.toString()); + // Assert.assertEquals(21, resultDec.integerDigitCount()); + resultDec = HiveDecimal.enforcePrecisionScale(resultDec, 38, 18); + Assert.assertTrue(resultDec == null); + + // Make sure zero trimming doesn't extend into the integer digits. + //--------------------------------------------------- + oldDec = HiveDecimalV1.create("199999999.99995"); + oldDec2 = HiveDecimalV1.create("100000000.00005"); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("300000000", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create("199999999.99995"); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create("100000000.00005"); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("300000000", resultDec.toString()); + Assert.assertEquals(9, resultDec.integerDigitCount()); + } + + @Test + public void testMinus() { + + HiveDecimalV1 oldDec; + HiveDecimalV1 oldDec2; + HiveDecimalV1 oldResultDec; + + HiveDecimal dec; + HiveDecimal dec1; + HiveDecimal dec2; + HiveDecimal resultDec; + + String decStr; + String decStr2; + + dec1 = HiveDecimal.create("3.140"); + dec1.validate(); + dec2 = HiveDecimal.create("1.00"); + dec2.validate(); + resultDec = dec1.add(dec2); + resultDec.validate(); + Assert.assertEquals("4.14", resultDec.toString()); + + decStr = "3.140"; + decStr2 = "1.00"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("2.14", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("2.14", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "3.140"; + decStr2 = "1.00000008733"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("2.13999991267", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("2.13999991267", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "3.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("2.13999999999999999991267", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("2.13999999999999999991267", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "30000000000.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("29999999999.13999999999999999991267", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("29999999999.13999999999999999991267", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "300000000000000.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("299999999999999.13999999999999999991267", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("299999999999999.13999999999999999991267", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + // Edge case? + decStr = "3000000000000000.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("2999999999999999.1399999999999999999127", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("2999999999999999.1399999999999999999127", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "300000000000000000000000.14"; + decStr2 = "0.0000055555555550008733"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("300000000000000000000000.13999444444444", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("300000000000000000000000.13999444444444", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "300000000000000000000000.14"; + decStr2 = "0.000005555555555000873355"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("300000000000000000000000.13999444444444", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("300000000000000000000000.13999444444444", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + // Example from HiveDecimal.subtract header comments. + decStr = "598575157855521918987423259.94094"; + decStr2 = "0.0000000000006711991169422033"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("598575157855521918987423259.94094", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("598575157855521918987423259.94094", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "598575157855521918987423259.94094"; + decStr2 = "0.5555555555556711991169422033"; + //--------------------------------------------------- + oldDec = HiveDecimalV1.create(decStr); + oldDec2 = HiveDecimalV1.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("598575157855521918987423259.38538444444", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("598575157855521918987423259.38538444444", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + } + + @Test + public void testSubtract() { + HiveDecimal dec1 = HiveDecimal.create("3.140"); + assertTrue(dec1 != null); + dec1.validate(); + HiveDecimal dec2 = HiveDecimal.create("1.00"); + assertTrue(dec2 != null); + dec2.validate(); + HiveDecimal result = dec1.subtract(dec2); + assertTrue(result != null); + result.validate(); + Assert.assertEquals("2.14", result.toString()); + + dec1 = HiveDecimal.create("0.00001786135888657847525803"); + assertTrue(dec1 != null); + dec1.validate(); + dec2 = HiveDecimal.create("3.0000123456789"); + assertTrue(dec2 != null); + dec2.validate(); + result = dec1.subtract(dec2); + assertTrue(result != null); + result.validate(); + Assert.assertEquals("-2.99999448432001342152474197", result.toString()); + } + + @Test + public void testPosMod() { + HiveDecimal hd1 = HiveDecimal.create("-100.91"); + assertTrue(hd1 != null); + hd1.validate(); + HiveDecimal hd2 = HiveDecimal.create("9.8"); + assertTrue(hd2 != null); + hd2.validate(); + HiveDecimal dec = hd1.remainder(hd2).add(hd2).remainder(hd2); + assertTrue(dec != null); + dec.validate(); + Assert.assertEquals("6.89", dec.toString()); + } + + @Test + public void testHashCode() { + Assert.assertEquals(HiveDecimal.create("9").hashCode(), HiveDecimal.create("9.00").hashCode()); + Assert.assertEquals(HiveDecimal.create("0").hashCode(), HiveDecimal.create("0.00").hashCode()); + } + + @Test + public void testException() { + HiveDecimal dec = HiveDecimal.create("3.1415.926"); + Assert.assertNull(dec); + dec = HiveDecimal.create("3abc43"); + Assert.assertNull(dec); + } + + @Test + public void testBinaryConversion() { + Random r = new Random(2399); + for (String decString : specialDecimalStrings) { + doTestBinaryConversion(decString, r); + } + } + + private void doTestBinaryConversion(String num, Random r) { + int scale = r.nextInt(HiveDecimal.MAX_SCALE); + HiveDecimal dec = HiveDecimal.create(num); + if (dec == null) { + return; + } + byte[] d = dec.setScale(scale).unscaledValue().toByteArray(); + HiveDecimal roundedDec = dec.setScale(scale, HiveDecimal.ROUND_HALF_UP); + Assert.assertEquals(roundedDec, HiveDecimal.create(new BigInteger(d), scale)); + } + +//------------------------------------------------------------------------------------------------ + + @Test + public void testDecimalsWithOneOne() { + doTestDecimalsWithPrecisionScale(decimal_1_1_txt, 1, 1); + } + + @Test + public void testDecimalsWithKv7Keys() { + doTestDecimalsWithPrecisionScale(kv7_txt_keys, 38, 18); + } + + public void doTestDecimalsWithPrecisionScale(String[] decStrings, int precision, int scale) { + + HiveDecimalV1 oldSum = HiveDecimalV1.create(0); + HiveDecimalWritable sum = new HiveDecimalWritable(0); + + for (int i = 0; i < decStrings.length; i++) { + + String string = decStrings[i]; + + HiveDecimalV1 oldDec = HiveDecimalV1.create(string); + + HiveDecimalV1 resultOldDec; + if (oldDec == null) { + resultOldDec = null; + } else { + resultOldDec = HiveDecimalV1.enforcePrecisionScale(oldDec, precision, scale); + } + + HiveDecimal dec = HiveDecimal.create(string); + + if (oldDec == null) { + Assert.assertTrue(dec == null); + continue; + } + HiveDecimal resultDec = HiveDecimal.enforcePrecisionScale(dec, precision, scale); + if (resultOldDec == null) { + Assert.assertTrue(resultDec == null); + continue; + } + + Assert.assertEquals(resultOldDec.toString(), resultDec.toString()); + Assert.assertEquals(resultOldDec.toFormatString(scale), resultDec.toFormatString(scale)); + + oldSum = oldSum.add(resultOldDec); + sum.mutateAdd(resultDec); + } + + Assert.assertEquals(oldSum.toString(), sum.toString()); + } + +//------------------------------------------------------------------------------------------------ + + @Test + public void testDecimalsWithOneOneWritable() { + doTestDecimalsWithPrecisionScaleWritable(decimal_1_1_txt, 1, 1); + } + + @Test + public void testDecimalsWithKv7KeysWritable() { + doTestDecimalsWithPrecisionScaleWritable(kv7_txt_keys, 38, 18); + } + + public void doTestDecimalsWithPrecisionScaleWritable(String[] decStrings, int precision, int scale) { + + HiveDecimalV1 oldSum = HiveDecimalV1.create(0); + HiveDecimalWritable sum = new HiveDecimalWritable(0); + + for (int i = 0; i < decStrings.length; i++) { + String string = decStrings[i]; + + HiveDecimalV1 oldDec = HiveDecimalV1.create(string); + HiveDecimalV1 resultOldDec; + if (oldDec == null) { + resultOldDec = null; + } else { + resultOldDec = HiveDecimalV1.enforcePrecisionScale(oldDec, precision, scale); + } + + HiveDecimalWritable decWritable = new HiveDecimalWritable(string); + if (oldDec == null) { + Assert.assertTrue(!decWritable.isSet()); + continue; + } + decWritable.mutateEnforcePrecisionScale(precision, scale);; + if (resultOldDec == null) { + Assert.assertTrue(!decWritable.isSet()); + continue; + } + + Assert.assertEquals(resultOldDec.toString(), decWritable.toString()); + Assert.assertEquals(resultOldDec.toFormatString(scale), decWritable.toFormatString(scale)); + + oldSum = oldSum.add(resultOldDec); + sum.mutateAdd(decWritable); + } + + Assert.assertEquals(oldSum.toString(), sum.toString()); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testSort() { + doTestSort(decimal_1_1_txt); + } + + @Test + public void testSortSpecial() { + doTestSort(specialDecimalStrings); + } + + @Test + public void testSortRandom() { + Random r = new Random(14434); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestSortRandom(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestSortRandom(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + public void doTestSortRandom(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + String[] randomStrings = new String[POUND_FACTOR]; + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + randomStrings[i] = bigDecimal.toString(); + } + + doTestSort(randomStrings); + } + + public void doTestSort(String[] decStrings) { + + HiveDecimalV1[] oldDecSortArray = new HiveDecimalV1[decStrings.length]; + HiveDecimal[] decSortArray = new HiveDecimal[decStrings.length]; + + int count = 0; + for (int i = 0; i < decStrings.length; i++) { + String string = decStrings[i]; + + HiveDecimalV1 oldDec = HiveDecimalV1.create(string); + if (oldDec == null) { + continue; + } + if (isTenPowerBug(oldDec.toString())) { + continue; + } + oldDecSortArray[count] = oldDec; + + HiveDecimal dec = HiveDecimal.create(string); + if (dec == null) { + Assert.fail(); + } + decSortArray[count] = dec; + count++; + } + + oldDecSortArray = Arrays.copyOf(oldDecSortArray, count); + decSortArray = Arrays.copyOf(decSortArray, count); + + Arrays.sort(oldDecSortArray); + Arrays.sort(decSortArray); + + for (int i = 0; i < count; i++) { + String oldDecString = oldDecSortArray[i].toString(); + String decString = decSortArray[i].toString(); + + if (!oldDecString.equals(decString)) { + Assert.fail(); + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCreateFromBigDecimal() { + Random r = new Random(14434); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomCreateFromBigDecimal(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomCreateFromBigDecimal(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomCreateFromBigDecimal(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestCreateFromBigDecimal(bigDecimal); + } + } + + @Test + public void testCreateFromBigDecimalSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestCreateFromBigDecimal(bigDecimal); + } + } + + private void doTestCreateFromBigDecimal(BigDecimal bigDecimal) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCreateFromBigDecimalNoRound() { + Random r = new Random(14434); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomCreateFromBigDecimalNoRound(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomCreateFromBigDecimalNoRound(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomCreateFromBigDecimalNoRound(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestCreateFromBigDecimalNoRound(bigDecimal); + } + } + + @Test + public void testCreateFromBigDecimalNoRoundSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestCreateFromBigDecimalNoRound(bigDecimal); + } + } + + private void doTestCreateFromBigDecimalNoRound(BigDecimal bigDecimal) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal, /* allowRounding */ false); + HiveDecimal dec = HiveDecimal.create(bigDecimal, /* allowRounding */ false); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + if (dec == null) { + Assert.fail(); + } + dec.validate(); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testCreateFromBigDecimalNegativeScaleSpecial() { + Random r = new Random(223965); + for (BigDecimal bigDecimal : specialBigDecimals) { + int negativeScale = -(0 + r.nextInt(38 + 1)); + bigDecimal = bigDecimal.setScale(negativeScale, BigDecimal.ROUND_HALF_UP); + doTestCreateFromBigDecimalNegativeScale(bigDecimal); + } + } + + private void doTestCreateFromBigDecimalNegativeScale(BigDecimal bigDecimal) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCreateFromBigInteger() { + doTestRandomCreateFromBigInteger(standardAlphabet); + } + + @Test + public void testRandomCreateFromBigIntegerSparse() { + for (String digitAlphabet : sparseAlphabets) { + doTestRandomCreateFromBigInteger(digitAlphabet); + } + } + + private void doTestRandomCreateFromBigInteger(String digitAlphabet) { + + Random r = new Random(11241); + for (int i = 0; i < POUND_FACTOR; i++) { + BigInteger bigInteger = randHiveBigInteger(r, digitAlphabet); + + doTestCreateFromBigInteger(bigInteger); + } + } + + @Test + public void testCreateFromBigIntegerSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestCreateFromBigInteger(bigDecimal.unscaledValue()); + } + } + + private void doTestCreateFromBigInteger(BigInteger bigInteger) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigInteger); + HiveDecimal dec = HiveDecimal.create(bigInteger); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCreateFromBigIntegerScale() { + doTestRandomCreateFromBigIntegerScale(standardAlphabet, false); + } + + @Test + public void testRandomCreateFromBigIntegerScaleFractionsOnly() { + doTestRandomCreateFromBigIntegerScale(standardAlphabet, true); + } + + @Test + public void testRandomCreateFromBigIntegerScaleSparse() { + for (String digitAlphabet : sparseAlphabets) { + doTestRandomCreateFromBigIntegerScale(digitAlphabet, false); + } + } + + private void doTestRandomCreateFromBigIntegerScale(String digitAlphabet, boolean fractionsOnly) { + + Random r = new Random(4448); + for (int i = 0; i < POUND_FACTOR; i++) { + BigInteger bigInteger = randHiveBigInteger(r, digitAlphabet); + + int scale; + if (fractionsOnly) { + scale = 1 + r.nextInt(38); + } else { + scale = 0 + r.nextInt(38 + 1); + } + + doTestCreateFromBigIntegerScale(bigInteger, scale); + } + } + + @Test + public void testCreateFromBigIntegerScaleSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestCreateFromBigIntegerScale(bigDecimal.unscaledValue(), bigDecimal.scale()); + } + } + + private void doTestCreateFromBigIntegerScale(BigInteger bigInteger, int scale) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigInteger, scale); + HiveDecimal dec = HiveDecimal.create(bigInteger, scale); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomSetFromDouble() { + Random r = new Random(14434); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomSetFromDouble(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomSetFromDouble(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomSetFromDouble(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestSetFromDouble(bigDecimal.doubleValue()); + } + } + + private void doTestRandomSetFromDouble() { + + Random r = new Random(94762); + for (int i = 0; i < POUND_FACTOR; i++) { + double randomDouble = r.nextDouble(); + + doTestSetFromDouble(randomDouble); + } + } + + @Test + public void testSetFromDoubleSpecial() { + + for (String specialString : specialDecimalStrings) { + double specialDouble = Double.valueOf(specialString); + doTestSetFromDouble(specialDouble); + } + } + + private void doTestSetFromDouble(double doubleValue) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(Double.toString(doubleValue)); + if (oldDec == null) { + return; + } + HiveDecimal dec = HiveDecimal.create(doubleValue); + if (dec == null) { + Assert.fail(); + } + dec.validate(); + if (!oldDec.toString().equals(dec.toString())) { + BigDecimal bigDecimal = new BigDecimal(dec.toString()); + for (int i = 16; i < 18;i++) { + BigDecimal trial = bigDecimal.setScale(i, HiveDecimal.ROUND_HALF_UP); + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCreateFromString() { + Random r = new Random(1221); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomCreateFromString(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomCreateFromString(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomCreateFromString(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestCreateFromString(bigDecimal); + } + } + + @Test + public void testCreateFromStringSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestCreateFromString(bigDecimal); + } + } + + private void doTestCreateFromString(BigDecimal bigDecimal) { + + String decString = bigDecimal.toPlainString(); + + HiveDecimalV1 oldDec = HiveDecimalV1.create(decString); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(decString); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCreateFromStringPadded() { + Random r = new Random(9774); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomCreateFromStringPadded(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomCreateFromStringPadded(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomCreateFromStringPadded(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestCreateFromStringPadded(bigDecimal); + } + } + + @Test + public void testCreateFromStringPaddedSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestCreateFromStringPadded(bigDecimal); + } + } + + private void doTestCreateFromStringPadded(BigDecimal bigDecimal) { + + String decString = bigDecimal.toPlainString(); + String decString1 = " " + decString; + String decString2 = decString + " "; + String decString3 = " " + decString + " "; + String decString4 = " " + decString; + String decString5 = decString + " "; + String decString6 = " " + decString + " "; + + HiveDecimalV1 oldDec; + HiveDecimal dec; + + oldDec = HiveDecimalV1.create(decString); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + + dec = HiveDecimal.create(decString1, true); + if (oldDec == null) { + assertTrue(dec == null); + } else { + assertTrue(dec != null); + dec.validate(); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + dec = HiveDecimal.create(decString2, true); + if (oldDec == null) { + assertTrue(dec == null); + } else { + assertTrue(dec != null); + dec.validate(); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + dec = HiveDecimal.create(decString3, true); + if (oldDec == null) { + assertTrue(dec == null); + } else { + assertTrue(dec != null); + dec.validate(); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + dec = HiveDecimal.create(decString4, true); + if (oldDec == null) { + assertTrue(dec == null); + } else { + assertTrue(dec != null); + dec.validate(); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + dec = HiveDecimal.create(decString5, true); + if (oldDec == null) { + assertTrue(dec == null); + } else { + assertTrue(dec != null); + dec.validate(); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + dec = HiveDecimal.create(decString6, true); + if (oldDec == null) { + assertTrue(dec == null); + } else { + assertTrue(dec != null); + dec.validate(); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCreateFromStringExponent() { + Random r = new Random(297111); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomCreateFromStringPadded(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomCreateFromStringPadded(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomCreateFromStringExponent(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestCreateFromStringExponent(bigDecimal); + } + } + + @Test + public void testCreateFromStringExponentSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestCreateFromStringExponent(bigDecimal); + } + } + + private void doTestCreateFromStringExponent(BigDecimal bigDecimal) { + + // Use toString which will have exponents instead of toPlainString. + String decString = bigDecimal.toString(); + + HiveDecimalV1 oldDec = HiveDecimalV1.create(decString); + HiveDecimal dec = HiveDecimal.create(decString); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomLongValue() { + Random r = new Random(73293); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomLongValue(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomLongValue(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomLongValue(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestLongValue(bigDecimal); + } + } + + @Test + public void testLongValueSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestLongValue(bigDecimal); + } + } + + private void doTestLongValue(BigDecimal bigDecimal) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + BigDecimal bigDecimalOldDec = oldDec.bigDecimalValue(); + BigDecimal bigDecimalDec = dec.bigDecimalValue(); + Assert.assertEquals(bigDecimalOldDec, bigDecimalDec); + + BigDecimal bigDecimalFloor = bigDecimalDec.setScale(0, BigDecimal.ROUND_DOWN); + long longValueBigDecimalFloor = bigDecimalFloor.longValue(); + boolean isLongExpected = + bigDecimalFloor.equals(bigDecimalDec.valueOf(longValueBigDecimalFloor)); + + boolean decIsLong = dec.isLong(); + long oldDecLong = oldDec.longValue(); + long decLong = dec.longValue(); + if (isLongExpected != decIsLong) { + Assert.fail(); + } + + if (decIsLong) { + if (oldDecLong != decLong) { + Assert.fail(); + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomIntValue() { + Random r = new Random(98333); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomIntValue(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomIntValue(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomIntValue(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestIntValue(bigDecimal); + } + } + + @Test + public void testIntValueSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestIntValue(bigDecimal); + } + } + + private void doTestIntValue(BigDecimal bigDecimal) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + BigDecimal bigDecimalOldDec = oldDec.bigDecimalValue(); + BigDecimal bigDecimalDec = dec.bigDecimalValue(); + Assert.assertEquals(bigDecimalOldDec, bigDecimalDec); + + BigDecimal bigDecimalFloor = bigDecimalDec.setScale(0, BigDecimal.ROUND_DOWN); + int intValueBigDecimalFloor = bigDecimalFloor.intValue(); + boolean isIntExpected = + bigDecimalFloor.equals(bigDecimalDec.valueOf(intValueBigDecimalFloor)); + + boolean decIsInt = dec.isInt(); + int oldDecInt = oldDec.intValue(); + int decInt = dec.intValue(); + if (isIntExpected != decIsInt) { + Assert.fail(); + } + + if (decIsInt) { + if (oldDecInt != decInt) { + Assert.fail(); + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomShortValue() { + Random r = new Random(15); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomShortValue(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomShortValue(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomShortValue(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestShortValue(bigDecimal); + } + } + + @Test + public void testShortValueSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestShortValue(bigDecimal); + } + } + + private void doTestShortValue(BigDecimal bigDecimal) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + BigDecimal bigDecimalOldDec = oldDec.bigDecimalValue(); + BigDecimal bigDecimalDec = dec.bigDecimalValue(); + Assert.assertEquals(bigDecimalOldDec, bigDecimalDec); + + BigDecimal bigDecimalFloor = bigDecimalDec.setScale(0, BigDecimal.ROUND_DOWN); + short shortValueBigDecimalFloor = bigDecimalFloor.shortValue(); + boolean isShortExpected = + bigDecimalFloor.equals(bigDecimalDec.valueOf(shortValueBigDecimalFloor)); + + boolean decIsShort = dec.isShort(); + short oldDecShort = oldDec.shortValue(); + short decShort = dec.shortValue(); + if (isShortExpected != decIsShort) { + Assert.fail(); + } + + if (decIsShort) { + if (oldDecShort != decShort) { + Assert.fail(); + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomByteValue() { + Random r = new Random(9292); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomByteValue(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomByteValue(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomByteValue(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestByteValue(bigDecimal); + } + } + + @Test + public void testByteValueSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestByteValue(bigDecimal); + } + } + + private void doTestByteValue(BigDecimal bigDecimal) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + BigDecimal bigDecimalOldDec = oldDec.bigDecimalValue(); + BigDecimal bigDecimalDec = dec.bigDecimalValue(); + Assert.assertEquals(bigDecimalOldDec, bigDecimalDec); + + BigDecimal bigDecimalFloor = bigDecimalDec.setScale(0, BigDecimal.ROUND_DOWN); + byte byteValueBigDecimalFloor = bigDecimalFloor.byteValue(); + boolean isByteExpected = + bigDecimalFloor.equals(bigDecimalDec.valueOf(byteValueBigDecimalFloor)); + + boolean decIsByte = dec.isByte(); + byte oldDecByte = oldDec.byteValue(); + byte decByte = dec.byteValue(); + if (isByteExpected != decIsByte) { + Assert.fail(); + } + + if (decIsByte) { + if (oldDecByte != decByte) { + Assert.fail(); + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomTimestamp() { + Random r = new Random(5476); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomTimestamp(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomTimestamp(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomTimestamp(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestTimestamp(bigDecimal); + } + } + + @Test + public void testTimestampSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestTimestamp(bigDecimal); + } + } + + private void doTestTimestamp(BigDecimal bigDecimal) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + Timestamp timestampOldDec = TimestampUtils.decimalToTimestamp(oldDec); + Timestamp timestampDec = TimestampUtils.decimalToTimestamp(dec); + if (timestampOldDec == null) { + Assert.assertTrue(timestampDec == null); + return; + } + if (timestampDec == null) { + return; + } + Assert.assertEquals(timestampOldDec, timestampDec); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomBigIntegerBytes() { + Random r = new Random(1050); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomBigIntegerBytes(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomBigIntegerBytes(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomBigIntegerBytes(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestBigIntegerBytes(bigDecimal); + } + } + + @Test + public void testBigIntegerBytesSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + int negativeScale = -(0 + r.nextInt(38 + 1)); + bigDecimal = bigDecimal.setScale(negativeScale, BigDecimal.ROUND_HALF_UP); + doTestBigIntegerBytes(bigDecimal); + } + } + + private void doTestBigIntegerBytes(BigDecimal bigDecimal) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + if (oldDec == null) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + assertTrue(dec != null); + dec.validate(); + + //--------------------------------------------------- + BigInteger oldBigInteger = oldDec.unscaledValue(); + int oldScale = oldDec.scale(); + //--------------------------------------------------- + + BigInteger bigInteger = dec.unscaledValue(); + int scale = dec.scale(); + + long[] scratchLongs = new long[HiveDecimal.SCRATCH_LONGS_LEN]; + byte[] scratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; + + int which = 0; + try { + which = 1; + int byteLength = dec.bigIntegerBytes(scratchLongs, scratchBuffer); + byte[] bytes = null; + if (byteLength == 0) { + Assert.fail(); + } else { + bytes = Arrays.copyOf(scratchBuffer, byteLength); + } + + which = 2; + byte[] bytesExpected = bigInteger.toByteArray(); + String bytesExpectedString = displayBytes(bytesExpected, 0, bytesExpected.length); + + if (!StringExpr.equal(bytes, 0, bytes.length, bytesExpected, 0, bytesExpected.length)) { + fail(); + } + + which = 3; + HiveDecimal createFromBigIntegerBytesDec = + HiveDecimal.createFromBigIntegerBytesAndScale( + bytes, 0, bytes.length, scale); + if (!createFromBigIntegerBytesDec.equals(dec)) { + fail(); + } + + } catch (Exception e) { + fail(); + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomToFormatString() { + Random r = new Random(1051); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomToFormatString(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomToFormatString(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomToFormatString(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestToFormatString(r, bigDecimal); + } + } + + @Test + public void testToFormatStringSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestToFormatString(r, bigDecimal); + } + } + + private void doTestToFormatString(Random r, BigDecimal bigDecimal) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec; + if (oldDec == null) { + dec = HiveDecimal.create(bigDecimal); + if (dec != null) { + Assert.fail(); + } + return; + } else { + dec = HiveDecimal.create(bigDecimal); + if (dec == null) { + if (isTenPowerBug(oldDec.toString())) { + return; + } + Assert.fail(); + } + } + dec.validate(); + + // UNDONE: Does this random range need to go as high as 38? + int formatScale = 0 + r.nextInt(38); + + String oldDecFormatString = oldDec.toFormatString(formatScale); + String decFormatString; + if (oldDecFormatString == null) { + decFormatString = dec.toFormatString(formatScale); + if (decFormatString != null) { + Assert.fail(); + } + return; + } else { + decFormatString = dec.toFormatString(formatScale); + if (decFormatString == null) { + if (isTenPowerBug(oldDecFormatString)) { + return; + } + Assert.fail(); + } + } + + if (!oldDecFormatString.equals(decFormatString)) { + fail(); + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomScaleByPowerOfTen() { + Random r = new Random(1052); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomScaleByPowerOfTen(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomScaleByPowerOfTen(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomScaleByPowerOfTen(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestScaleByPowerOfTen(r, bigDecimal); + } + } + + @Test + public void testScaleByPowerOfTenSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestScaleByPowerOfTen(r, bigDecimal); + } + } + + private void doTestScaleByPowerOfTen(Random r, BigDecimal bigDecimal) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + HiveDecimalV1 oldPowerDec; + HiveDecimal powerDec; + + for (int power = -(2 * HiveDecimal.MAX_SCALE + 1); + power <= 2 * HiveDecimal.MAX_SCALE + 1; + power++) { + + oldPowerDec = oldDec.scaleByPowerOfTen(power); + boolean isEqual; + if (oldPowerDec == null) { + powerDec = dec.scaleByPowerOfTen(power); + if (powerDec != null) { + Assert.fail(); + } + return; + } else { + String oldPowerDecString = oldPowerDec.toString(); + powerDec = dec.scaleByPowerOfTen(power); + if (powerDec == null) { + if (isTenPowerBug(oldPowerDec.toString())) { + return; + } + Assert.fail(); + continue; + } + powerDec.validate(); + String powerDecString = powerDec.toString(); + isEqual = oldPowerDecString.equals(powerDecString); + if (!isEqual) { + if (oldPowerDecString.equals("0.00000000000000000000000000000000000001") || + oldPowerDecString.equals("-0.00000000000000000000000000000000000001")) { + continue; + } + Assert.fail(); + } + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomWriteReadFields() throws Exception { + Random r = new Random(1052); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomWriteReadFields(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomWriteReadFields(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomWriteReadFields(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) throws Exception { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestWriteReadFields(r, bigDecimal); + } + } + + @Test + public void testWriteReadFieldsSpecial() throws Exception { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestWriteReadFields(r, bigDecimal); + } + } + + private void doTestWriteReadFields(Random r, BigDecimal bigDecimal) throws IOException { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(baos); + + HiveDecimalWritable decimalWritableOut = new HiveDecimalWritable(dec); + decimalWritableOut.write(out); + + byte[] valueBytes = baos.toByteArray(); + + ByteArrayInputStream bais = new ByteArrayInputStream(valueBytes); + DataInputStream in = new DataInputStream(bais); + HiveDecimalWritable decimalWritableIn = new HiveDecimalWritable(); + decimalWritableIn.readFields(in); + + Assert.assertEquals(dec, decimalWritableIn.getHiveDecimal()); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomBigIntegerBytesScaled() throws Exception { + Random r = new Random(1052); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomBigIntegerBytesScaled(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomBigIntegerBytesScaled(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomBigIntegerBytesScaled(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) throws Exception { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestBigIntegerBytesScaled(r, bigDecimal); + } + } + + @Test + public void testBigIntegerBytesScaledSpecial() throws Exception { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestBigIntegerBytesScaled(r, bigDecimal); + } + } + + private void doTestBigIntegerBytesScaled(Random r, BigDecimal bigDecimal) throws IOException { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + int scale = oldDec.scale(); + int newScale; + if (scale == HiveDecimal.MAX_SCALE) { + newScale = scale; + } else { + newScale = scale + r.nextInt(HiveDecimal.MAX_SCALE - scale); + } + + HiveDecimalV1 oldDecScaled = oldDec.setScale(newScale); + HiveDecimalWritableV1 oldDecScaledWritable = new HiveDecimalWritableV1(oldDecScaled); + + byte[] bytesExpected = oldDecScaledWritable.getInternalStorage(); + + byte[] bytes = dec.bigIntegerBytesScaled(newScale); + + if (!StringExpr.equal(bytes, 0, bytes.length, bytesExpected, 0, bytesExpected.length)) { + Assert.fail(); + } + + HiveDecimalWritableV1 oldDecWritableRetrieve = new HiveDecimalWritableV1(bytesExpected, newScale); + HiveDecimalV1 oldDecRetrieve = oldDecWritableRetrieve.getHiveDecimal(); + Assert.assertTrue(oldDecRetrieve != null); + + HiveDecimal decRetrieve = HiveDecimal.createFromBigIntegerBytesAndScale(bytes, newScale); + Assert.assertTrue(decRetrieve != null); + + Assert.assertEquals(oldDecRetrieve.toString(), decRetrieve.toString()); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomRoundFloor() { + Random r = new Random(1052); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomRound(r, standardAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_FLOOR); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomRound(r, sparseAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_FLOOR); + } + } + } + + private void doTestRandomRound(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor, int roundingMode) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestRound(r, bigDecimal, roundingMode); + } + } + + @Test + public void testRoundFloorSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestRound(r, bigDecimal, HiveDecimal.ROUND_FLOOR); + } + } + + // Used by all flavors. + private void doTestRound(Random r, BigDecimal bigDecimal, int roundingMode) { + + // Temporarily.... + bigDecimal = bigDecimal.abs(); + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + HiveDecimalV1 oldScaledDec; + HiveDecimal scaledDec; + + for (int newScale = -(2 * HiveDecimal.MAX_SCALE + 1); + newScale <= 2 * HiveDecimal.MAX_SCALE + 1; + newScale++) { + + oldScaledDec = oldDec.setScale(newScale, roundingMode); + boolean isEqual; + if (oldScaledDec == null) { + scaledDec = dec.setScale(newScale, roundingMode); + if (scaledDec != null) { + Assert.fail(); + } + return; + } else { + scaledDec = dec.setScale(newScale, roundingMode); + if (scaledDec == null) { + if (isTenPowerBug(oldScaledDec.toString())) { + continue; + } + Assert.fail(); + } + scaledDec.validate(); + isEqual = oldScaledDec.toString().equals(scaledDec.toString()); + if (!isEqual) { + Assert.fail(); + } + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomRoundCeiling() { + Random r = new Random(1053); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomRound(r, standardAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_CEILING); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomRound(r, sparseAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_CEILING); + } + } + } + + @Test + public void testRoundCeilingSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestRound(r, bigDecimal, HiveDecimal.ROUND_CEILING); + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomRoundHalfUp() { + Random r = new Random(1053); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomRound(r, standardAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_HALF_UP); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomRound(r, sparseAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_HALF_UP); + } + } + } + + @Test + public void testRoundHalfUpSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestRound(r, bigDecimal, HiveDecimal.ROUND_HALF_UP); + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomRoundHalfEven() { + Random r = new Random(1053); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomRound(r, standardAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_HALF_EVEN); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomRound(r, sparseAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_HALF_EVEN); + } + } + } + + @Test + public void testRoundHalfEvenSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestRound(r, bigDecimal, HiveDecimal.ROUND_HALF_EVEN); + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCompareTo() { + Random r = new Random(1054); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomCompareTo(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomCompareTo(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomCompareTo(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (BigDecimalPairFlavor bigDecimalPairFlavor : BigDecimalPairFlavor.values()) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal[] pair = randHiveBigDecimalPair(r, digitAlphabet, bigDecimalFlavor, bigDecimalPairFlavor); + + doTestCompareTo(r, pair[0], pair[1]); + } + } + } + + @Test + public void testCompareToSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + for (BigDecimal bigDecimal2 : specialBigDecimals) { + doTestCompareTo(r, bigDecimal, bigDecimal2); + } + } + } + + private void doTestCompareTo(Random r, BigDecimal bigDecimal, BigDecimal bigDecimal2) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + HiveDecimalV1 oldDec2 = HiveDecimalV1.create(bigDecimal2); + if (oldDec2 != null && isTenPowerBug(oldDec2.toString())) { + return; + } + HiveDecimal dec2 = HiveDecimal.create(bigDecimal2); + if (oldDec2 == null) { + assertTrue(dec2 == null); + return; + } + assertTrue(dec2 != null); + dec.validate(); + + // Verify. + Assert.assertEquals(oldDec.toString(), dec.toString()); + Assert.assertEquals(oldDec2.toString(), dec2.toString()); + + int oldCompareTo; + int compareTo; + + // Same object. + oldCompareTo = oldDec.compareTo(oldDec); + Assert.assertEquals(0, oldCompareTo); + compareTo = dec.compareTo(dec); + Assert.assertEquals(0, compareTo); + + // Two objects. + oldCompareTo = oldDec.compareTo(oldDec2); + compareTo = dec.compareTo(dec2); + Assert.assertEquals(oldCompareTo, compareTo); + + int oldCompareToReverse = oldDec2.compareTo(oldDec); + int compareToReverse = dec2.compareTo(dec); + Assert.assertEquals(oldCompareToReverse, compareToReverse); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomAdd() { + Random r = new Random(1055); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomAdd(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomAdd(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomAdd(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (BigDecimalPairFlavor bigDecimalPairFlavor : BigDecimalPairFlavor.values()) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal[] pair = randHiveBigDecimalPair(r, digitAlphabet, bigDecimalFlavor, bigDecimalPairFlavor); + + doTestAdd(r, pair[0], pair[1]); + } + } + } + + @Test + public void testAddSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + for (BigDecimal bigDecimal2 : specialBigDecimals) { + doTestAdd(r, bigDecimal, bigDecimal2); + } + } + } + + private void doTestAdd(Random r, BigDecimal bigDecimal, BigDecimal bigDecimal2) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + HiveDecimalV1 oldDec2 = HiveDecimalV1.create(bigDecimal2); + if (oldDec2 != null && isTenPowerBug(oldDec2.toString())) { + return; + } + HiveDecimal dec2 = HiveDecimal.create(bigDecimal2); + if (oldDec2 == null) { + assertTrue(dec2 == null); + return; + } + assertTrue(dec2 != null); + dec.validate(); + + // Verify. + Assert.assertEquals(oldDec.toString(), dec.toString()); + Assert.assertEquals(oldDec2.toString(), dec2.toString()); + + // Add to self. + HiveDecimalV1 oldAddDec; + HiveDecimal addDec; + + oldAddDec = oldDec.add(oldDec); + boolean isEqual; + if (oldAddDec == null) { + addDec = dec.add(dec); + assertTrue(addDec == null); + return; + } else { + addDec = dec.add(dec); + if (addDec == null) { + if (isTenPowerBug(oldAddDec.toString())) { + return; + } + Assert.fail(); + } + addDec.validate(); + isEqual = oldAddDec.toString().equals(addDec.toString()); + if (!isEqual) { + Assert.fail(); + } + } + + // Add two decimals. + oldAddDec = oldDec.add(oldDec2); + if (oldAddDec == null) { + addDec = dec.add(dec2); + assertTrue(addDec == null); + return; + } else { + addDec = dec.add(dec2); + if (addDec == null) { + if (isTenPowerBug(oldAddDec.toString())) { + return; + } + Assert.fail(); + } + addDec.validate(); + isEqual = oldAddDec.toString().equals(addDec.toString()); + if (!isEqual) { + Assert.fail(); + } + } + + // Add negative self. + + oldAddDec = oldDec.add(oldDec.negate()); + if (oldAddDec == null) { + addDec = dec.add(dec.negate()); + assertTrue(addDec == null); + return; + } else { + addDec = dec.add(dec.negate()); + if (addDec == null) { + if (isTenPowerBug(oldAddDec.toString())) { + return; + } + Assert.fail(); + } + addDec.validate(); + isEqual = oldAddDec.toString().equals(addDec.toString()); + if (!isEqual) { + Assert.fail(); + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomSubtract() { + Random r = new Random(1055); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomSubtract(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomSubtract(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomSubtract(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (BigDecimalPairFlavor bigDecimalPairFlavor : BigDecimalPairFlavor.values()) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal[] pair = randHiveBigDecimalPair(r, digitAlphabet, bigDecimalFlavor, bigDecimalPairFlavor); + + doTestSubtract(r, pair[0], pair[1]); + } + } + } + + @Test + public void testSubtractSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + for (BigDecimal bigDecimal2 : specialBigDecimals) { + doTestSubtract(r, bigDecimal, bigDecimal2); + } + } + } + + private void doTestSubtract(Random r, BigDecimal bigDecimal, BigDecimal bigDecimal2) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + HiveDecimalV1 oldDec2 = HiveDecimalV1.create(bigDecimal2); + if (oldDec2 != null && isTenPowerBug(oldDec2.toString())) { + return; + } + HiveDecimal dec2 = HiveDecimal.create(bigDecimal2); + if (oldDec2 == null) { + assertTrue(dec2 == null); + return; + } + assertTrue(dec2 != null); + dec.validate(); + + // Verify. + Assert.assertEquals(oldDec.toString(), dec.toString()); + Assert.assertEquals(oldDec2.toString(), dec2.toString()); + + // Subtract from self. + HiveDecimalV1 oldSubtractDec; + HiveDecimal subtractDec; + + oldSubtractDec = oldDec.subtract(oldDec); + Assert.assertEquals(0, oldSubtractDec.signum()); + subtractDec = dec.subtract(dec); + Assert.assertEquals(0, subtractDec.signum()); + + boolean isEqual; + oldSubtractDec = oldDec.subtract(oldDec2); + if (oldSubtractDec == null) { + subtractDec = dec.subtract(dec2); + assertTrue(subtractDec == null); + return; + } else { + subtractDec = dec.subtract(dec2); + if (subtractDec == null) { + if (isTenPowerBug(oldSubtractDec.toString())) { + return; + } + Assert.fail(); + } + subtractDec.validate(); + isEqual = oldSubtractDec.toString().equals(subtractDec.toString()); + if (!isEqual) { + Assert.fail(); + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomMultiply() { + Random r = new Random(1056); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomMultiply(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomMultiply(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomMultiply(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (BigDecimalPairFlavor bigDecimalPairFlavor : BigDecimalPairFlavor.values()) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal[] pair = randHiveBigDecimalPair(r, digitAlphabet, bigDecimalFlavor, bigDecimalPairFlavor); + + doTestMultiply(r, pair[0], pair[1]); + } + } + } + + @Test + public void testMultiplySpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + for (BigDecimal bigDecimal2 : specialBigDecimals) { + doTestMultiply(r, bigDecimal, bigDecimal2); + } + } + } + + private void doTestMultiply(Random r, BigDecimal bigDecimal, BigDecimal bigDecimal2) { + + HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + HiveDecimalV1 oldDec2 = HiveDecimalV1.create(bigDecimal2); + if (oldDec2 != null && isTenPowerBug(oldDec2.toString())) { + return; + } + HiveDecimal dec2 = HiveDecimal.create(bigDecimal2); + if (oldDec2 == null) { + assertTrue(dec2 == null); + return; + } + assertTrue(dec2 != null); + dec.validate(); + + // Verify. + Assert.assertEquals(oldDec.toString(), dec.toString()); + Assert.assertEquals(oldDec2.toString(), dec2.toString()); + + // Multiply by self. + BigDecimal bigDecimalMultiply = bigDecimal.multiply(bigDecimal); + BigDecimal bigDecimalMultiplyAbs = bigDecimalMultiply.abs(); + String bigDecimalMultiplyAbsString = bigDecimalMultiplyAbs.toString(); + int digits = bigDecimalMultiplyAbsString.indexOf('.') != -1 ? bigDecimalMultiplyAbsString.length() - 1: bigDecimalMultiplyAbsString.length(); + HiveDecimalV1 oldMultiplyDec; + HiveDecimal multiplyDec; + + oldMultiplyDec = oldDec.multiply(oldDec); + boolean isEqual; + if (oldMultiplyDec == null) { + multiplyDec = dec.multiply(dec); + if (multiplyDec != null) { + Assert.fail(); + } + return; + } else { + multiplyDec = dec.multiply(dec); + if (multiplyDec == null) { + if (isTenPowerBug(oldMultiplyDec.toString())) { + return; + } + Assert.fail(); + } + multiplyDec.validate(); + isEqual = oldMultiplyDec.toString().equals(multiplyDec.toString()); + if (!isEqual) { + Assert.fail(); + } + } + + bigDecimalMultiply = bigDecimal.multiply(bigDecimal2); + bigDecimalMultiplyAbs = bigDecimalMultiply.abs(); + bigDecimalMultiplyAbsString = bigDecimalMultiplyAbs.toString(); + digits = bigDecimalMultiplyAbsString.indexOf('.') != -1 ? bigDecimalMultiplyAbsString.length() - 1: bigDecimalMultiplyAbsString.length(); + oldMultiplyDec = oldDec.multiply(oldDec2); + if (oldMultiplyDec == null) { + multiplyDec = dec.multiply(dec2); + if (multiplyDec != null) { + Assert.fail(); + } + return; + } else { + multiplyDec = dec.multiply(dec2); + if (multiplyDec == null) { + if (isTenPowerBug(oldMultiplyDec.toString())) { + return; + } + Assert.fail(); + } + multiplyDec.validate(); + isEqual = oldMultiplyDec.toString().equals(multiplyDec.toString()); + if (!isEqual) { + Assert.fail(); + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomDecimal64() { + Random r = new Random(2497); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomDecimal64(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomDecimal64(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomDecimal64(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestDecimal64(r, bigDecimal); + } + } + + @Test + public void testDecimal64Special() { + Random r = new Random(198); + for (BigDecimal bigDecimal : specialBigDecimals) { + int precision = Math.min(bigDecimal.precision(), HiveDecimalWritable.DECIMAL64_DECIMAL_DIGITS); + int scale = Math.min(bigDecimal.scale(), precision); + doTestDecimal64(r, bigDecimal, precision, scale); + } + } + + private void doTestDecimal64(Random r, BigDecimal inputBigDecimal) { + final int precision = 1 + r.nextInt(HiveDecimalWritable.DECIMAL64_DECIMAL_DIGITS); + assertTrue(HiveDecimalWritable.isPrecisionDecimal64(precision)); + final int scale = r.nextInt(precision + 1); + + doTestDecimal64(r, inputBigDecimal, precision, scale); + } + + private void doTestDecimal64(Random r, BigDecimal inputBigDecimal, int precision, int scale) { + + BigDecimal bigDecimal = inputBigDecimal; + + if (!bigDecimal.equals(BigDecimal.ZERO)) { + while (true) { + bigDecimal = bigDecimal.remainder(BigDecimal.valueOf(10).pow(precision - scale)); + bigDecimal = bigDecimal.setScale(scale, BigDecimal.ROUND_DOWN); + if (!bigDecimal.unscaledValue().equals(BigInteger.ZERO)) { + break; + } + bigDecimal = randHiveBigDecimalNormalRange(r, standardAlphabet); + } + } + + HiveDecimal dec = HiveDecimal.create(bigDecimal); + assertTrue(dec != null); + dec.validate(); + + HiveDecimalWritable decWritable = new HiveDecimalWritable(dec); + + final long decimal64Long = decWritable.serialize64(scale); + assertTrue(decimal64Long <= HiveDecimalWritable.getDecimal64AbsMax(precision)); + HiveDecimalWritable resultWritable = new HiveDecimalWritable(0); + resultWritable.deserialize64(decimal64Long, scale); + + assertEquals(dec, resultWritable.getHiveDecimal()); + } + + //------------------------------------------------------------------------------------------------ + + public static String displayBytes(byte[] bytes, int start, int length) { + StringBuilder sb = new StringBuilder(); + for (int i = start; i < start + length; i++) { + sb.append(String.format("\\%03d", (int) (bytes[i] & 0xff))); + } + return sb.toString(); + } +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimalVersion.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimalVersion.java new file mode 100644 index 0000000000000000000000000000000000000000..cf80b0c81d3091653f8903bb7c2080c61713f24b --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimalVersion.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.Field; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Arrays; +import java.util.TreeSet; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; + +import org.apache.hadoop.hive.VersionTestBase; +import org.junit.*; + +import static org.junit.Assert.*; + +public class TestHiveDecimalVersion extends VersionTestBase { + + /* + * Validation: + * 1) Substitute class name for "ThisClass". + * 2) Only public fields and methods are versioned. + * 3) Methods compare on [non-]static, return type, name, parameter types, exceptions thrown. + * 4) Fields compare on [non-]static, type, name, value when static + */ + @Test + public void testVerifyHiveDecimalPublicMethodsAndFieldsVersions() throws IllegalAccessException { + + Map versionedClassToNameMap = new HashMap(); + versionedClassToNameMap.put(HiveDecimalV1.class, "HiveDecimal"); + versionedClassToNameMap.put(HiveDecimal.class, "HiveDecimal"); + + doVerifyVersions( + HiveDecimalV1.class, HiveDecimalVersionV1.class, + HiveDecimal.class, HiveDecimalVersionV2.class, + versionedClassToNameMap); + + } +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestBytesColumnVector.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestBytesColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..e14abf1006a4aa0e4c83f1e5a914abccbc92d3a1 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestBytesColumnVector.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + +import java.util.Arrays; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class TestBytesColumnVector { + @Test + public void testSmallBufferReuse() { + BytesColumnVector col = new BytesColumnVector(); + int smallWriteSize = 1024; + int largeWriteSize = 1024 * 1024 * 2; + + int rowIdx = 0; + int bytesWrittenToBytes1 = 0; + col.reset(); + + // Initial write (small value) + byte[] bytes1 = writeToBytesColumnVector(rowIdx, col, smallWriteSize, (byte) 1); + bytesWrittenToBytes1 += smallWriteSize; + + // Write a large value. This should use a different byte buffer + rowIdx++; + byte[] bytes2 = writeToBytesColumnVector(rowIdx, col, largeWriteSize, (byte) 2); + assertFalse(bytes1 == bytes2); + + // Another small write. smallBuffer should be re-used for this write + rowIdx++; + byte[] bytes3 = writeToBytesColumnVector(rowIdx, col, smallWriteSize, (byte) 1); + bytesWrittenToBytes1 += smallWriteSize; + assertTrue(bytes1 == bytes3); + + // Write another large value. This should use a different byte buffer + rowIdx++; + byte[] bytes4 = writeToBytesColumnVector(rowIdx, col, largeWriteSize, (byte) 3); + assertFalse(bytes1 == bytes4); + assertFalse(bytes2 == bytes4); + + // Eventually enough small writes should result in another buffer getting created + boolean gotNewBuffer = false; + // Test is dependent on getting a new buffer within 1MB. + // This may need to change as the implementation changes. + for (int i = 0; i < 1024; ++i) { + rowIdx++; + byte[] currBytes = writeToBytesColumnVector(rowIdx, col, smallWriteSize, (byte) 1); + if (currBytes == bytes1) { + bytesWrittenToBytes1 += smallWriteSize; + } else { + gotNewBuffer = true; + break; + } + } + + assertTrue(gotNewBuffer); + + // All small writes to the first buffer should be in contiguous memory + for (int i = 0; i < bytesWrittenToBytes1; ++i) { + assertEquals((byte) 1, bytes1[i]); + } + } + + // Write a value to the column vector, and return back the byte buffer used. + private static byte[] writeToBytesColumnVector(int rowIdx, BytesColumnVector col, int writeSize, byte val) { + col.ensureValPreallocated(writeSize); + byte[] bytes = col.getValPreallocatedBytes(); + int startIdx = col.getValPreallocatedStart(); + Arrays.fill(bytes, startIdx, startIdx + writeSize, val); + col.setValPreallocated(rowIdx, writeSize); + return bytes; + } +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestListColumnVector.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestListColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..395d8f5d7520420bd2178f5ab9ee8eab4d2c6487 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestListColumnVector.java @@ -0,0 +1,200 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Test for ListColumnVector + */ +public class TestListColumnVector { + + @Test + public void testFlatten() throws Exception { + LongColumnVector col1 = new LongColumnVector(10); + ListColumnVector vector = new ListColumnVector(10, col1); + vector.init(); + + // TEST - repeating NULL & no selection + col1.isRepeating = true; + vector.isRepeating = true; + vector.noNulls = false; + vector.isNull[0] = true; + vector.childCount = 0; + for(int i=0; i < 10; ++i) { + col1.vector[i] = i + 3; + vector.offsets[i] = i; + vector.lengths[i] = 10 + i; + } + vector.flatten(false, null, 10); + // make sure the vector was flattened + assertFalse(vector.isRepeating); + assertFalse(vector.noNulls); + // child isn't flattened, because parent is repeating null + assertTrue(col1.isRepeating); + assertTrue(col1.noNulls); + for(int i=0; i < 10; ++i) { + assertTrue("isNull at " + i, vector.isNull[i]); + } + for(int i=0; i < 10; ++i) { + StringBuilder buf = new StringBuilder(); + vector.stringifyValue(buf, i); + assertEquals("null", buf.toString()); + } + vector.unFlatten(); + assertTrue(col1.isRepeating); + assertTrue(vector.isRepeating); + + // TEST - repeating NULL & selection + Arrays.fill(vector.isNull, 1, 10, false); + int[] sel = new int[]{3, 5, 7}; + vector.flatten(true, sel, 3); + for(int i=1; i < 10; i++) { + assertEquals("failure at " + i, + i == 3 || i == 5 || i == 7, vector.isNull[i]); + } + vector.unFlatten(); + + // TEST - repeating non-NULL & no-selection + vector.noNulls = true; + vector.isRepeating = true; + vector.offsets[0] = 0; + vector.lengths[0] = 3; + vector.childCount = 3; + vector.flatten(false, null, 10); + // make sure the vector was flattened + assertFalse(vector.isRepeating); + assertFalse(vector.noNulls); + assertFalse(col1.isRepeating); + assertFalse(col1.noNulls); + for(int i=0; i < 10; ++i) { + assertEquals("offset at " + i, 0, vector.offsets[i]); + assertEquals("length at " + i, 3, vector.lengths[i]); + } + for(int i=0; i < 10; ++i) { + StringBuilder buf = new StringBuilder(); + vector.stringifyValue(buf, i); + assertEquals("[3, 3, 3]", buf.toString()); + } + vector.unFlatten(); + assertTrue(col1.isRepeating); + assertTrue(col1.noNulls); + assertTrue(vector.isRepeating); + assertTrue(vector.noNulls); + + // TEST - repeating non-NULL & selection + Arrays.fill(vector.offsets, 1, 10, -1); + Arrays.fill(vector.lengths, 1, 10, -1); + Arrays.fill(col1.vector, 1, 10, -1); + vector.flatten(true, sel, 3); + for(int i=1; i < 10; i++) { + if (i == 3 || i == 5 || i == 7) { + assertEquals("failure at " + i, 0, vector.offsets[i]); + assertEquals("failure at " + i, 3, vector.lengths[i]); + } else { + assertEquals("failure at " + i, -1, vector.offsets[i]); + assertEquals("failure at " + i, -1, vector.lengths[i]); + } + } + for(int i=0; i < 3; ++i) { + assertEquals("failure at " + i, 3, col1.vector[i]); + } + for(int i=3; i < 10; ++i) { + assertEquals("failure at " + i, -1, col1.vector[i]); + } + vector.unFlatten(); + + // TEST - reset + vector.reset(); + assertFalse(col1.isRepeating); + assertTrue(col1.noNulls); + assertFalse(vector.isRepeating); + assertTrue(vector.noNulls); + assertEquals(0, vector.childCount); + } + + @Test + public void testSet() throws Exception { + LongColumnVector input1 = new LongColumnVector(10); + ListColumnVector input = new ListColumnVector(10, input1); + input.init(); + LongColumnVector output1 = new LongColumnVector(30); + ListColumnVector output = new ListColumnVector(10, output1); + output.init(); + input.noNulls = false; + input.isNull[6] = true; + input.childCount = 11; + Arrays.fill(output1.vector, -1); + for(int i=0; i < 10; ++i) { + input1.vector[i] = 10 * i; + input.offsets[i] = i; + input.lengths[i] = 2; + output.offsets[i] = i + 2; + output.lengths[i] = 3; + } + output.childCount = 30; + + // copy a null + output.setElement(3, 6, input); + assertEquals(30, output.childCount); + StringBuilder buf = new StringBuilder(); + output.stringifyValue(buf, 3); + assertEquals("null", buf.toString()); + + // copy a value + output.setElement(3, 5, input); + assertEquals(30, output.offsets[3]); + assertEquals(2, output.lengths[3]); + assertEquals(32, output.childCount); + buf = new StringBuilder(); + output.stringifyValue(buf, 3); + assertEquals("[50, 60]", buf.toString()); + + // overwrite a value + output.setElement(3, 4, input); + assertEquals(34, output.childCount); + assertEquals(34, output1.vector.length); + assertEquals(50, output1.vector[30]); + assertEquals(60, output1.vector[31]); + buf = new StringBuilder(); + output.stringifyValue(buf, 3); + assertEquals("[40, 50]", buf.toString()); + + input.reset(); + assertEquals(false, input1.isRepeating); + assertEquals(true, input.noNulls); + output.reset(); + assertEquals(0, output.childCount); + + input.isRepeating = true; + input.offsets[0] = 0; + input.lengths[0] = 10; + output.setElement(2, 7, input); + assertEquals(10, output.childCount); + buf = new StringBuilder(); + output.stringifyValue(buf, 2); + assertEquals("[0, 10, 20, 30, 40, 50, 60, 70, 80, 90]", buf.toString()); + } +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestMapColumnVector.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestMapColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..c77c286d54256829f6c95715f5f1bb4ed83c6bb5 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestMapColumnVector.java @@ -0,0 +1,224 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Test for MapColumnVector + */ +public class TestMapColumnVector { + + @Test + public void testFlatten() throws Exception { + LongColumnVector col1 = new LongColumnVector(10); + DoubleColumnVector col2 = new DoubleColumnVector(10); + MapColumnVector vector = new MapColumnVector(10, col1, col2); + vector.init(); + + // TEST - repeating NULL & no selection + col1.isRepeating = true; + vector.isRepeating = true; + vector.noNulls = false; + vector.isNull[0] = true; + vector.childCount = 0; + for(int i=0; i < 10; ++i) { + col1.vector[i] = i + 3; + col2.vector[i] = i * 10; + vector.offsets[i] = i; + vector.lengths[i] = 10 + i; + } + vector.flatten(false, null, 10); + // make sure the vector was flattened + assertFalse(vector.isRepeating); + assertFalse(vector.noNulls); + // child isn't flattened, because parent is repeating null + assertTrue(col1.isRepeating); + assertTrue(col1.noNulls); + for(int i=0; i < 10; ++i) { + assertTrue("isNull at " + i, vector.isNull[i]); + } + for(int i=0; i < 10; ++i) { + StringBuilder buf = new StringBuilder(); + vector.stringifyValue(buf, i); + assertEquals("null", buf.toString()); + } + vector.unFlatten(); + assertTrue(col1.isRepeating); + assertTrue(vector.isRepeating); + + // TEST - repeating NULL & selection + Arrays.fill(vector.isNull, 1, 10, false); + int[] sel = new int[]{3, 5, 7}; + vector.flatten(true, sel, 3); + for(int i=1; i < 10; i++) { + assertEquals("failure at " + i, + i == 3 || i == 5 || i == 7, vector.isNull[i]); + } + vector.unFlatten(); + + // TEST - repeating non-NULL & no-selection + vector.noNulls = true; + vector.isRepeating = true; + vector.offsets[0] = 0; + vector.lengths[0] = 3; + vector.childCount = 3; + vector.flatten(false, null, 10); + // make sure the vector was flattened + assertFalse(vector.isRepeating); + assertFalse(vector.noNulls); + assertFalse(col1.isRepeating); + assertFalse(col1.noNulls); + assertFalse(col2.isRepeating); + assertFalse(col2.noNulls); + for(int i=0; i < 10; ++i) { + assertEquals("offset at " + i, 0, vector.offsets[i]); + assertEquals("length at " + i, 3, vector.lengths[i]); + } + for(int i=0; i < 10; ++i) { + StringBuilder buf = new StringBuilder(); + vector.stringifyValue(buf, i); + assertEquals("[{\"key\": 3, \"value\": 0.0}," + + " {\"key\": 3, \"value\": 10.0}," + + " {\"key\": 3, \"value\": 20.0}]", buf.toString()); + } + vector.unFlatten(); + assertTrue(col1.isRepeating); + assertTrue(col1.noNulls); + assertTrue(vector.isRepeating); + assertFalse(col2.isRepeating); + assertTrue(col2.noNulls); + assertTrue(vector.noNulls); + + // TEST - repeating non-NULL & selection + Arrays.fill(vector.offsets, 1, 10, -1); + Arrays.fill(vector.lengths, 1, 10, -1); + Arrays.fill(col1.vector, 1, 10, -1); + vector.flatten(true, sel, 3); + for(int i=1; i < 10; i++) { + if (i == 3 || i == 5 || i == 7) { + assertEquals("failure at " + i, 0, vector.offsets[i]); + assertEquals("failure at " + i, 3, vector.lengths[i]); + } else { + assertEquals("failure at " + i, -1, vector.offsets[i]); + assertEquals("failure at " + i, -1, vector.lengths[i]); + } + } + for(int i=0; i < 3; ++i) { + assertEquals("failure at " + i, 3, col1.vector[i]); + } + for(int i=3; i < 10; ++i) { + assertEquals("failure at " + i, -1, col1.vector[i]); + } + vector.unFlatten(); + + // TEST - reset + vector.reset(); + assertFalse(col1.isRepeating); + assertTrue(col1.noNulls); + assertFalse(col2.isRepeating); + assertTrue(col2.noNulls); + assertFalse(vector.isRepeating); + assertTrue(vector.noNulls); + assertEquals(0, vector.childCount); + } + + @Test + public void testSet() throws Exception { + LongColumnVector input1 = new LongColumnVector(10); + DoubleColumnVector input2 = new DoubleColumnVector(10); + MapColumnVector input = new MapColumnVector(10, input1, input2); + input.init(); + LongColumnVector output1 = new LongColumnVector(30); + DoubleColumnVector output2 = new DoubleColumnVector(30); + MapColumnVector output = new MapColumnVector(10, output1, output2); + output.init(); + input.noNulls = false; + input.isNull[6] = true; + input.childCount = 11; + Arrays.fill(output1.vector, -1); + for(int i=0; i < 10; ++i) { + input1.vector[i] = 10 * i; + input2.vector[i] = 100 * i; + input.offsets[i] = i; + input.lengths[i] = 2; + output.offsets[i] = i + 2; + output.lengths[i] = 3; + } + output.childCount = 30; + + // copy a null + output.setElement(3, 6, input); + assertEquals(30, output.childCount); + StringBuilder buf = new StringBuilder(); + output.stringifyValue(buf, 3); + assertEquals("null", buf.toString()); + + // copy a value + output.setElement(3, 5, input); + assertEquals(30, output.offsets[3]); + assertEquals(2, output.lengths[3]); + assertEquals(32, output.childCount); + buf = new StringBuilder(); + output.stringifyValue(buf, 3); + assertEquals("[{\"key\": 50, \"value\": 500.0}," + + " {\"key\": 60, \"value\": 600.0}]", buf.toString()); + + // overwrite a value + output.setElement(3, 4, input); + assertEquals(34, output.childCount); + assertEquals(34, output1.vector.length); + assertEquals(50, output1.vector[30]); + assertEquals(60, output1.vector[31]); + buf = new StringBuilder(); + output.stringifyValue(buf, 3); + assertEquals("[{\"key\": 40, \"value\": 400.0}," + + " {\"key\": 50, \"value\": 500.0}]", buf.toString()); + + input.reset(); + assertEquals(false, input1.isRepeating); + assertEquals(true, input.noNulls); + output.reset(); + assertEquals(0, output.childCount); + + input.isRepeating = true; + input.offsets[0] = 0; + input.lengths[0] = 10; + output.setElement(2, 7, input); + assertEquals(10, output.childCount); + buf = new StringBuilder(); + output.stringifyValue(buf, 2); + assertEquals("[{\"key\": 0, \"value\": 0.0}," + + " {\"key\": 10, \"value\": 100.0}," + + " {\"key\": 20, \"value\": 200.0}," + + " {\"key\": 30, \"value\": 300.0}," + + " {\"key\": 40, \"value\": 400.0}," + + " {\"key\": 50, \"value\": 500.0}," + + " {\"key\": 60, \"value\": 600.0}," + + " {\"key\": 70, \"value\": 700.0}," + + " {\"key\": 80, \"value\": 800.0}," + + " {\"key\": 90, \"value\": 900.0}]", buf.toString()); + } +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..c175ed08c73e4c8d177b0015ddabae413f6f4c21 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.sql.Timestamp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Test for StructColumnVector + */ +public class TestStructColumnVector { + + @Test + public void testFlatten() throws Exception { + LongColumnVector col1 = new LongColumnVector(10); + LongColumnVector col2 = new LongColumnVector(10); + StructColumnVector vector = new StructColumnVector(10, col1, col2); + vector.init(); + col1.isRepeating = true; + for(int i=0; i < 10; ++i) { + col1.vector[i] = i; + col2.vector[i] = 2 * i; + } + vector.flatten(false, null, 10); + assertFalse(col1.isRepeating); + for(int i=0; i < 10; ++i) { + assertEquals("col1 at " + i, 0, col1.vector[i]); + assertEquals("col2 at " + i, 2 * i, col2.vector[i]); + } + vector.unFlatten(); + assertTrue(col1.isRepeating); + for(int i=0; i < 10; ++i) { + StringBuilder buf = new StringBuilder(); + vector.stringifyValue(buf, i); + assertEquals("[0, " + (2 * i) + "]", buf.toString()); + } + vector.reset(); + assertFalse(col1.isRepeating); + } + + @Test + public void testSet() throws Exception { + LongColumnVector input1 = new LongColumnVector(10); + LongColumnVector input2 = new LongColumnVector(10); + StructColumnVector input = new StructColumnVector(10, input1, input2); + input.init(); + LongColumnVector output1 = new LongColumnVector(10); + LongColumnVector output2 = new LongColumnVector(10); + StructColumnVector output = new StructColumnVector(10, output1, output2); + output.init(); + input1.isRepeating = true; + input2.noNulls = false; + input2.isNull[5] = true; + input.noNulls = false; + input.isNull[6] = true; + for(int i=0; i < 10; ++i) { + input1.vector[i] = i + 1; + input2.vector[i] = i + 2; + } + output.setElement(3, 6, input); + StringBuilder buf = new StringBuilder(); + output.stringifyValue(buf, 3); + assertEquals("null", buf.toString()); + output.setElement(3, 5, input); + buf = new StringBuilder(); + output.stringifyValue(buf, 3); + assertEquals("[1, null]", buf.toString()); + output.setElement(3, 4, input); + buf = new StringBuilder(); + output.stringifyValue(buf, 3); + assertEquals("[1, 6]", buf.toString()); + input.reset(); + assertEquals(false, input1.isRepeating); + assertEquals(true, input.noNulls); + } + + @Test + public void testStringify() throws IOException { + VectorizedRowBatch batch = new VectorizedRowBatch(2); + LongColumnVector x1 = new LongColumnVector(); + TimestampColumnVector x2 = new TimestampColumnVector(); + StructColumnVector x = new StructColumnVector(1024, x1, x2); + BytesColumnVector y = new BytesColumnVector(); + batch.cols[0] = x; + batch.cols[1] = y; + batch.reset(); + Timestamp ts = Timestamp.valueOf("2000-01-01 00:00:00"); + for(int r=0; r < 10; ++r) { + batch.size += 1; + x1.vector[r] = 3 * r; + ts.setTime(ts.getTime() + 1000); + x2.set(r, ts); + byte[] buffer = ("value " + r).getBytes(StandardCharsets.UTF_8); + y.setRef(r, buffer, 0, buffer.length); + } + final String EXPECTED = ("Column vector types: 0:STRUCT, 1:BYTES\n" + + "[[0, 2000-01-01 00:00:01.0], \"value 0\"]\n" + + "[[3, 2000-01-01 00:00:02.0], \"value 1\"]\n" + + "[[6, 2000-01-01 00:00:03.0], \"value 2\"]\n" + + "[[9, 2000-01-01 00:00:04.0], \"value 3\"]\n" + + "[[12, 2000-01-01 00:00:05.0], \"value 4\"]\n" + + "[[15, 2000-01-01 00:00:06.0], \"value 5\"]\n" + + "[[18, 2000-01-01 00:00:07.0], \"value 6\"]\n" + + "[[21, 2000-01-01 00:00:08.0], \"value 7\"]\n" + + "[[24, 2000-01-01 00:00:09.0], \"value 8\"]\n" + + "[[27, 2000-01-01 00:00:10.0], \"value 9\"]"); + assertEquals(EXPECTED, batch.toString()); + } +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..6e5d5c8cfb8a8d33eba568e8180c5620652b0ec2 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + +import org.junit.Test; + +import java.io.PrintWriter; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.sql.Timestamp; +import java.util.Date; +import java.util.Random; + +import org.apache.hadoop.hive.common.type.RandomTypeUtil; + +import static org.junit.Assert.*; + +/** + * Test for ListColumnVector + */ +public class TestTimestampColumnVector { + + private static int TEST_COUNT = 5000; + + private static int fake = 0; + + @Test + public void testSaveAndRetrieve() throws Exception { + + Random r = new Random(1234); + TimestampColumnVector timestampColVector = new TimestampColumnVector(); + Timestamp[] randTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE]; + + for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { + Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r); + randTimestamps[i] = randTimestamp; + timestampColVector.set(i, randTimestamp); + } + for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { + Timestamp retrievedTimestamp = timestampColVector.asScratchTimestamp(i); + Timestamp randTimestamp = randTimestamps[i]; + if (!retrievedTimestamp.equals(randTimestamp)) { + assertTrue(false); + } + } + } + + @Test + public void testTimestampCompare() throws Exception { + Random r = new Random(1234); + TimestampColumnVector timestampColVector = new TimestampColumnVector(); + Timestamp[] randTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE]; + Timestamp[] candTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE]; + int[] compareToLeftRights = new int[VectorizedRowBatch.DEFAULT_SIZE]; + int[] compareToRightLefts = new int[VectorizedRowBatch.DEFAULT_SIZE]; + + for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { + Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r); + randTimestamps[i] = randTimestamp; + timestampColVector.set(i, randTimestamp); + Timestamp candTimestamp = RandomTypeUtil.getRandTimestamp(r); + candTimestamps[i] = candTimestamp; + compareToLeftRights[i] = candTimestamp.compareTo(randTimestamp); + compareToRightLefts[i] = randTimestamp.compareTo(candTimestamp); + } + + for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { + Timestamp retrievedTimestamp = timestampColVector.asScratchTimestamp(i); + Timestamp randTimestamp = randTimestamps[i]; + if (!retrievedTimestamp.equals(randTimestamp)) { + assertTrue(false); + } + Timestamp candTimestamp = candTimestamps[i]; + int compareToLeftRight = timestampColVector.compareTo(candTimestamp, i); + if (compareToLeftRight != compareToLeftRights[i]) { + assertTrue(false); + } + int compareToRightLeft = timestampColVector.compareTo(i, candTimestamp); + if (compareToRightLeft != compareToRightLefts[i]) { + assertTrue(false); + } + } + } + + /* + @Test + public void testGenerate() throws Exception { + PrintWriter writer = new PrintWriter("/Users/you/timestamps.txt"); + Random r = new Random(18485); + for (int i = 0; i < 25; i++) { + Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r); + writer.println(randTimestamp.toString()); + } + for (int i = 0; i < 25; i++) { + Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r, 1965, 2025); + writer.println(randTimestamp.toString()); + } + writer.close(); + } + */ +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestUnionColumnVector.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestUnionColumnVector.java new file mode 100644 index 0000000000000000000000000000000000000000..c378cd41a46bbb3d242547db944dacf90dde9e56 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestUnionColumnVector.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Test for StructColumnVector + */ +public class TestUnionColumnVector { + + @Test + public void testFlatten() throws Exception { + LongColumnVector col1 = new LongColumnVector(10); + LongColumnVector col2 = new LongColumnVector(10); + UnionColumnVector vector = new UnionColumnVector(10, col1, col2); + vector.init(); + col1.isRepeating = true; + for(int i=0; i < 10; ++i) { + vector.tags[i] = i % 2; + col1.vector[i] = i; + col2.vector[i] = 2 * i; + } + vector.flatten(false, null, 10); + assertFalse(col1.isRepeating); + for(int i=0; i < 10; ++i) { + assertEquals(i % 2, vector.tags[i]); + assertEquals("col1 at " + i, 0, col1.vector[i]); + assertEquals("col2 at " + i, 2 * i, col2.vector[i]); + } + vector.unFlatten(); + assertTrue(col1.isRepeating); + for(int i=0; i < 10; ++i) { + StringBuilder buf = new StringBuilder(); + vector.stringifyValue(buf, i); + assertEquals("{\"tag\": " + (i % 2) + ", \"value\": " + + (i % 2 == 0 ? 0 : 2 * i) + "}", buf.toString()); + } + vector.reset(); + assertFalse(col1.isRepeating); + } + + @Test + public void testSet() throws Exception { + LongColumnVector input1 = new LongColumnVector(10); + LongColumnVector input2 = new LongColumnVector(10); + UnionColumnVector input = new UnionColumnVector(10, input1, input2); + input.init(); + LongColumnVector output1 = new LongColumnVector(10); + LongColumnVector output2 = new LongColumnVector(10); + UnionColumnVector output = new UnionColumnVector(10, output1, output2); + output.init(); + input1.isRepeating = true; + for(int i=0; i < 10; ++i) { + input.tags[i] = i % 2; + input1.vector[i] = i + 1; + input2.vector[i] = i + 2; + } + output.setElement(3, 4, input); + StringBuilder buf = new StringBuilder(); + output.stringifyValue(buf, 3); + assertEquals("{\"tag\": 0, \"value\": 1}", buf.toString()); + input.noNulls = false; + input.isNull[5] = true; + output.setElement(3, 5, input); + buf = new StringBuilder(); + output.stringifyValue(buf, 3); + assertEquals("null", buf.toString()); + input.reset(); + assertEquals(false, input1.isRepeating); + assertEquals(true, input.noNulls); + } +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestStringExpr.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestStringExpr.java new file mode 100644 index 0000000000000000000000000000000000000000..63c210ad4a8f75f65da2fe449891bc4ee33b4970 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestStringExpr.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.*; + +public class TestStringExpr { + @Test + public void test() throws Exception { + StringExpr.Finder pattern = compile("pattern"); + assertNotNull(pattern); + + StringExpr.Finder patternOneChar = compile("g"); + assertNotNull(patternOneChar); + + StringExpr.Finder patternZero = compile(""); + assertNotNull(patternZero); + + String input1 = "string that contains a patterN..."; + String input2 = "string that contains a pattern..."; + String input3 = "pattern at the start of a string"; + String input4 = "string that ends with a pattern"; + + assertEquals("Testing invalid match", -1, find(pattern, input1)); + assertEquals("Testing valid match", 23, find(pattern, input2)); + assertEquals("Testing single-character match", 5, find(patternOneChar, input1)); + assertEquals("Testing zero-length pattern", 0, find(patternZero, input1)); + assertEquals("Testing match at start of string", 0, find(pattern, input3)); + assertEquals("Testing match at end of string", 24, find(pattern, input4)); + } + + private StringExpr.Finder compile(String pattern) { + return StringExpr.compile(pattern.getBytes(StandardCharsets.UTF_8)); + } + + private int find(StringExpr.Finder finder, String string) { + byte[] bytes = string.getBytes(StandardCharsets.UTF_8); + return finder.find(bytes, 0, bytes.length); + } +} \ No newline at end of file diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/util/TestJavaDataModel.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/util/TestJavaDataModel.java new file mode 100644 index 0000000000000000000000000000000000000000..9295b899acb7cad38aa52720f83612688177ce14 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/ql/util/TestJavaDataModel.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.util; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; + +public final class TestJavaDataModel { + + private static final String DATA_MODEL_PROPERTY = "sun.arch.data.model"; + + private String previousModelSetting; + + @Before + public void setUp() throws Exception { + previousModelSetting = System.getProperty(DATA_MODEL_PROPERTY); + } + + @After + public void tearDown() throws Exception { + if (previousModelSetting != null) { + System.setProperty(DATA_MODEL_PROPERTY, previousModelSetting); + } else { + System.clearProperty(DATA_MODEL_PROPERTY); + } + } + + @Test + public void testGetDoesNotReturnNull() throws Exception { + JavaDataModel model = JavaDataModel.get(); + assertNotNull(model); + } + + @Test + public void testGetModelForSystemWhenSetTo32() throws Exception { + System.setProperty(DATA_MODEL_PROPERTY, "32"); + assertSame(JavaDataModel.JAVA32, JavaDataModel.getModelForSystem()); + } + + @Test + public void testGetModelForSystemWhenSetTo64() throws Exception { + System.setProperty(DATA_MODEL_PROPERTY, "64"); + assertSame(JavaDataModel.JAVA64, JavaDataModel.getModelForSystem()); + } + + @Test + public void testGetModelForSystemWhenSetToUnknown() throws Exception { + System.setProperty(DATA_MODEL_PROPERTY, "unknown"); + assertSame(JavaDataModel.JAVA64, JavaDataModel.getModelForSystem()); + } + + @Test + public void testGetModelForSystemWhenUndefined() throws Exception { + System.clearProperty(DATA_MODEL_PROPERTY); + assertSame(JavaDataModel.JAVA64, JavaDataModel.getModelForSystem()); + } +} \ No newline at end of file diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java new file mode 100644 index 0000000000000000000000000000000000000000..b8f5472edd78e1c66a24368285ff20ccb47b891a --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.io; + +import org.junit.*; +import static org.junit.Assert.*; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; + +import org.apache.hadoop.hive.common.type.HiveDecimal; + +/** + * Unit tests for tsting the fast allocation-free conversion + * between HiveDecimalWritable and Decimal128 + */ +public class TestHiveDecimalWritable { + + @Test + public void testHiveDecimalWritable() { + + HiveDecimalWritable decWritable; + + HiveDecimal nullDec = null; + decWritable = new HiveDecimalWritable(nullDec); + assertTrue(!decWritable.isSet()); + decWritable = new HiveDecimalWritable("1"); + assertTrue(decWritable.isSet()); + + // UNDONE: more! + } +} + diff --git a/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritableVersion.java b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritableVersion.java new file mode 100644 index 0000000000000000000000000000000000000000..8e0f396fe0ae240c6df2dda1e31a768976132e83 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritableVersion.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.io; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.Field; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Arrays; +import java.util.TreeSet; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; + +import org.apache.hadoop.hive.VersionTestBase; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveDecimalV1; +import org.junit.*; + +import static org.junit.Assert.*; + +public class TestHiveDecimalWritableVersion extends VersionTestBase { + + /* + * Validation: + * 1) Substitute class name for "ThisClass". + * 2) Only public fields and methods are versioned. + * 3) Methods compare on [non-]static, return type, name, parameter types, exceptions thrown. + * 4) Fields compare on [non-]static, type, name, value when static + */ + @Test + public void testVerifyHiveDecimalWritablePublicMethodsAndFieldsVersions() throws IllegalAccessException { + + Map versionedClassToNameMap = new HashMap(); + versionedClassToNameMap.put(HiveDecimalV1.class, "HiveDecimal"); + versionedClassToNameMap.put(HiveDecimal.class, "HiveDecimal"); + versionedClassToNameMap.put(HiveDecimalWritableV1.class, "HiveDecimalWritable"); + versionedClassToNameMap.put(HiveDecimalWritable.class, "HiveDecimalWritable"); + + doVerifyVersions( + HiveDecimalWritableV1.class, HiveDecimalWritableVersionV1.class, + HiveDecimalWritable.class, HiveDecimalWritableVersionV2.class, + versionedClassToNameMap); + + } +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hive/common/util/TestBloomFilter.java b/storage-api/hive-storage-api/src/test/org/apache/hive/common/util/TestBloomFilter.java new file mode 100644 index 0000000000000000000000000000000000000000..cd1fa08f9a929c7885cc508fe11d6635d62a236d --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hive/common/util/TestBloomFilter.java @@ -0,0 +1,715 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.common.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.Random; + +import org.junit.Assert; +import org.junit.Test; + +/** + * + */ +public class TestBloomFilter { + private static final int COUNT = 100; + Random rand = new Random(123); + + @Test(expected = IllegalArgumentException.class) + public void testBloomIllegalArg1() { + BloomFilter bf = new BloomFilter(0, 0); + } + + @Test(expected = IllegalArgumentException.class) + public void testBloomIllegalArg2() { + BloomFilter bf = new BloomFilter(0, 0.1); + } + + @Test(expected = IllegalArgumentException.class) + public void testBloomIllegalArg3() { + BloomFilter bf = new BloomFilter(1, 0.0); + } + + @Test(expected = IllegalArgumentException.class) + public void testBloomIllegalArg4() { + BloomFilter bf = new BloomFilter(1, 1.0); + } + + @Test(expected = IllegalArgumentException.class) + public void testBloomIllegalArg5() { + BloomFilter bf = new BloomFilter(-1, -1); + } + + + @Test + public void testBloomNumBits() { + assertEquals(0, BloomFilter.optimalNumOfBits(0, 0)); + assertEquals(0, BloomFilter.optimalNumOfBits(0, 1)); + assertEquals(0, BloomFilter.optimalNumOfBits(1, 1)); + assertEquals(7, BloomFilter.optimalNumOfBits(1, 0.03)); + assertEquals(72, BloomFilter.optimalNumOfBits(10, 0.03)); + assertEquals(729, BloomFilter.optimalNumOfBits(100, 0.03)); + assertEquals(7298, BloomFilter.optimalNumOfBits(1000, 0.03)); + assertEquals(72984, BloomFilter.optimalNumOfBits(10000, 0.03)); + assertEquals(729844, BloomFilter.optimalNumOfBits(100000, 0.03)); + assertEquals(7298440, BloomFilter.optimalNumOfBits(1000000, 0.03)); + assertEquals(6235224, BloomFilter.optimalNumOfBits(1000000, 0.05)); + assertEquals(1870567268, BloomFilter.optimalNumOfBits(300000000, 0.05)); + assertEquals(1437758756, BloomFilter.optimalNumOfBits(300000000, 0.1)); + assertEquals(432808512, BloomFilter.optimalNumOfBits(300000000, 0.5)); + assertEquals(1393332198, BloomFilter.optimalNumOfBits(3000000000L, 0.8)); + assertEquals(657882327, BloomFilter.optimalNumOfBits(3000000000L, 0.9)); + assertEquals(0, BloomFilter.optimalNumOfBits(3000000000L, 1)); + } + + @Test + public void testBloomNumHashFunctions() { + assertEquals(1, BloomFilter.optimalNumOfHashFunctions(-1, -1)); + assertEquals(1, BloomFilter.optimalNumOfHashFunctions(0, 0)); + assertEquals(1, BloomFilter.optimalNumOfHashFunctions(10, 0)); + assertEquals(1, BloomFilter.optimalNumOfHashFunctions(10, 10)); + assertEquals(7, BloomFilter.optimalNumOfHashFunctions(10, 100)); + assertEquals(1, BloomFilter.optimalNumOfHashFunctions(100, 100)); + assertEquals(1, BloomFilter.optimalNumOfHashFunctions(1000, 100)); + assertEquals(1, BloomFilter.optimalNumOfHashFunctions(10000, 100)); + assertEquals(1, BloomFilter.optimalNumOfHashFunctions(100000, 100)); + assertEquals(1, BloomFilter.optimalNumOfHashFunctions(1000000, 100)); + } + + @Test + public void testBloomFilterBytes() { + BloomFilter bf = new BloomFilter(10000); + byte[] val = new byte[]{1, 2, 3}; + byte[] val1 = new byte[]{1, 2, 3, 4}; + byte[] val2 = new byte[]{1, 2, 3, 4, 5}; + byte[] val3 = new byte[]{1, 2, 3, 4, 5, 6}; + + assertEquals(false, bf.test(val)); + assertEquals(false, bf.test(val1)); + assertEquals(false, bf.test(val2)); + assertEquals(false, bf.test(val3)); + bf.add(val); + assertEquals(true, bf.test(val)); + assertEquals(false, bf.test(val1)); + assertEquals(false, bf.test(val2)); + assertEquals(false, bf.test(val3)); + bf.add(val1); + assertEquals(true, bf.test(val)); + assertEquals(true, bf.test(val1)); + assertEquals(false, bf.test(val2)); + assertEquals(false, bf.test(val3)); + bf.add(val2); + assertEquals(true, bf.test(val)); + assertEquals(true, bf.test(val1)); + assertEquals(true, bf.test(val2)); + assertEquals(false, bf.test(val3)); + bf.add(val3); + assertEquals(true, bf.test(val)); + assertEquals(true, bf.test(val1)); + assertEquals(true, bf.test(val2)); + assertEquals(true, bf.test(val3)); + + byte[] randVal = new byte[COUNT]; + for (int i = 0; i < COUNT; i++) { + rand.nextBytes(randVal); + bf.add(randVal); + } + // last value should be present + assertEquals(true, bf.test(randVal)); + // most likely this value should not exist + randVal[0] = 0; + randVal[1] = 0; + randVal[2] = 0; + randVal[3] = 0; + randVal[4] = 0; + assertEquals(false, bf.test(randVal)); + + assertEquals(7800, bf.sizeInBytes()); + } + + @Test + public void testBloomFilterByte() { + BloomFilter bf = new BloomFilter(10000); + byte val = Byte.MIN_VALUE; + byte val1 = 1; + byte val2 = 2; + byte val3 = Byte.MAX_VALUE; + + assertEquals(false, bf.testLong(val)); + assertEquals(false, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val); + assertEquals(true, bf.testLong(val)); + assertEquals(false, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val1); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val2); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(true, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val3); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(true, bf.testLong(val2)); + assertEquals(true, bf.testLong(val3)); + + byte randVal = 0; + for (int i = 0; i < COUNT; i++) { + randVal = (byte) rand.nextInt(Byte.MAX_VALUE); + bf.addLong(randVal); + } + // last value should be present + assertEquals(true, bf.testLong(randVal)); + // most likely this value should not exist + assertEquals(false, bf.testLong((byte) -120)); + + assertEquals(7800, bf.sizeInBytes()); + } + + @Test + public void testBloomFilterInt() { + BloomFilter bf = new BloomFilter(10000); + int val = Integer.MIN_VALUE; + int val1 = 1; + int val2 = 2; + int val3 = Integer.MAX_VALUE; + + assertEquals(false, bf.testLong(val)); + assertEquals(false, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val); + assertEquals(true, bf.testLong(val)); + assertEquals(false, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val1); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val2); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(true, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val3); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(true, bf.testLong(val2)); + assertEquals(true, bf.testLong(val3)); + + int randVal = 0; + for (int i = 0; i < COUNT; i++) { + randVal = rand.nextInt(); + bf.addLong(randVal); + } + // last value should be present + assertEquals(true, bf.testLong(randVal)); + // most likely this value should not exist + assertEquals(false, bf.testLong(-120)); + + assertEquals(7800, bf.sizeInBytes()); + } + + @Test + public void testBloomFilterLong() { + BloomFilter bf = new BloomFilter(10000); + long val = Long.MIN_VALUE; + long val1 = 1; + long val2 = 2; + long val3 = Long.MAX_VALUE; + + assertEquals(false, bf.testLong(val)); + assertEquals(false, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val); + assertEquals(true, bf.testLong(val)); + assertEquals(false, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val1); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val2); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(true, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val3); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(true, bf.testLong(val2)); + assertEquals(true, bf.testLong(val3)); + + long randVal = 0; + for (int i = 0; i < COUNT; i++) { + randVal = rand.nextLong(); + bf.addLong(randVal); + } + // last value should be present + assertEquals(true, bf.testLong(randVal)); + // most likely this value should not exist + assertEquals(false, bf.testLong(-120)); + + assertEquals(7800, bf.sizeInBytes()); + } + + @Test + public void testBloomFilterFloat() { + BloomFilter bf = new BloomFilter(10000); + float val = Float.MIN_VALUE; + float val1 = 1.1f; + float val2 = 2.2f; + float val3 = Float.MAX_VALUE; + + assertEquals(false, bf.testDouble(val)); + assertEquals(false, bf.testDouble(val1)); + assertEquals(false, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val); + assertEquals(true, bf.testDouble(val)); + assertEquals(false, bf.testDouble(val1)); + assertEquals(false, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val1); + assertEquals(true, bf.testDouble(val)); + assertEquals(true, bf.testDouble(val1)); + assertEquals(false, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val2); + assertEquals(true, bf.testDouble(val)); + assertEquals(true, bf.testDouble(val1)); + assertEquals(true, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val3); + assertEquals(true, bf.testDouble(val)); + assertEquals(true, bf.testDouble(val1)); + assertEquals(true, bf.testDouble(val2)); + assertEquals(true, bf.testDouble(val3)); + + float randVal = 0; + for (int i = 0; i < COUNT; i++) { + randVal = rand.nextFloat(); + bf.addDouble(randVal); + } + // last value should be present + assertEquals(true, bf.testDouble(randVal)); + // most likely this value should not exist + assertEquals(false, bf.testDouble(-120.2f)); + + assertEquals(7800, bf.sizeInBytes()); + } + + @Test + public void testBloomFilterDouble() { + BloomFilter bf = new BloomFilter(10000); + double val = Double.MIN_VALUE; + double val1 = 1.1d; + double val2 = 2.2d; + double val3 = Double.MAX_VALUE; + + assertEquals(false, bf.testDouble(val)); + assertEquals(false, bf.testDouble(val1)); + assertEquals(false, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val); + assertEquals(true, bf.testDouble(val)); + assertEquals(false, bf.testDouble(val1)); + assertEquals(false, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val1); + assertEquals(true, bf.testDouble(val)); + assertEquals(true, bf.testDouble(val1)); + assertEquals(false, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val2); + assertEquals(true, bf.testDouble(val)); + assertEquals(true, bf.testDouble(val1)); + assertEquals(true, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val3); + assertEquals(true, bf.testDouble(val)); + assertEquals(true, bf.testDouble(val1)); + assertEquals(true, bf.testDouble(val2)); + assertEquals(true, bf.testDouble(val3)); + + double randVal = 0; + for (int i = 0; i < COUNT; i++) { + randVal = rand.nextDouble(); + bf.addDouble(randVal); + } + // last value should be present + assertEquals(true, bf.testDouble(randVal)); + // most likely this value should not exist + assertEquals(false, bf.testDouble(-120.2d)); + + assertEquals(7800, bf.sizeInBytes()); + } + + @Test + public void testBloomFilterString() { + BloomFilter bf = new BloomFilter(100000); + String val = "bloo"; + String val1 = "bloom fil"; + String val2 = "bloom filter"; + String val3 = "cuckoo filter"; + + assertEquals(false, bf.testString(val)); + assertEquals(false, bf.testString(val1)); + assertEquals(false, bf.testString(val2)); + assertEquals(false, bf.testString(val3)); + bf.addString(val); + assertEquals(true, bf.testString(val)); + assertEquals(false, bf.testString(val1)); + assertEquals(false, bf.testString(val2)); + assertEquals(false, bf.testString(val3)); + bf.addString(val1); + assertEquals(true, bf.testString(val)); + assertEquals(true, bf.testString(val1)); + assertEquals(false, bf.testString(val2)); + assertEquals(false, bf.testString(val3)); + bf.addString(val2); + assertEquals(true, bf.testString(val)); + assertEquals(true, bf.testString(val1)); + assertEquals(true, bf.testString(val2)); + assertEquals(false, bf.testString(val3)); + bf.addString(val3); + assertEquals(true, bf.testString(val)); + assertEquals(true, bf.testString(val1)); + assertEquals(true, bf.testString(val2)); + assertEquals(true, bf.testString(val3)); + + long randVal = 0; + for (int i = 0; i < COUNT; i++) { + randVal = rand.nextLong(); + bf.addString(Long.toString(randVal)); + } + // last value should be present + assertEquals(true, bf.testString(Long.toString(randVal))); + // most likely this value should not exist + assertEquals(false, bf.testString(Long.toString(-120))); + + assertEquals(77944, bf.sizeInBytes()); + } + + @Test + public void testMerge() { + BloomFilter bf = new BloomFilter(10000); + String val = "bloo"; + String val1 = "bloom fil"; + String val2 = "bloom filter"; + String val3 = "cuckoo filter"; + bf.addString(val); + bf.addString(val1); + bf.addString(val2); + bf.addString(val3); + + BloomFilter bf2 = new BloomFilter(10000); + String v = "2_bloo"; + String v1 = "2_bloom fil"; + String v2 = "2_bloom filter"; + String v3 = "2_cuckoo filter"; + bf2.addString(v); + bf2.addString(v1); + bf2.addString(v2); + bf2.addString(v3); + + assertEquals(true, bf.testString(val)); + assertEquals(true, bf.testString(val1)); + assertEquals(true, bf.testString(val2)); + assertEquals(true, bf.testString(val3)); + assertEquals(false, bf.testString(v)); + assertEquals(false, bf.testString(v1)); + assertEquals(false, bf.testString(v2)); + assertEquals(false, bf.testString(v3)); + + bf.merge(bf2); + + assertEquals(true, bf.testString(val)); + assertEquals(true, bf.testString(val1)); + assertEquals(true, bf.testString(val2)); + assertEquals(true, bf.testString(val3)); + assertEquals(true, bf.testString(v)); + assertEquals(true, bf.testString(v1)); + assertEquals(true, bf.testString(v2)); + assertEquals(true, bf.testString(v3)); + } + + @Test + public void testSerialize() throws Exception { + BloomFilter bf1 = new BloomFilter(10000); + String[] inputs = { + "bloo", + "bloom fil", + "bloom filter", + "cuckoo filter", + }; + + for (String val : inputs) { + bf1.addString(val); + } + + // Serialize/deserialize + ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); + BloomFilter.serialize(bytesOut, bf1); + ByteArrayInputStream bytesIn = new ByteArrayInputStream(bytesOut.toByteArray()); + BloomFilter bf2 = BloomFilter.deserialize(bytesIn); + + for (String val : inputs) { + assertEquals("Testing bf1 with " + val, true, bf1.testString(val)); + assertEquals("Testing bf2 with " + val, true, bf2.testString(val)); + } + } + + @Test + public void testMergeBloomFilterBytes() throws Exception { + BloomFilter bf1 = new BloomFilter(10000); + BloomFilter bf2 = new BloomFilter(10000); + + String[] inputs1 = { + "bloo", + "bloom fil", + "bloom filter", + "cuckoo filter", + }; + + String[] inputs2 = { + "2_bloo", + "2_bloom fil", + "2_bloom filter", + "2_cuckoo filter", + }; + + for (String val : inputs1) { + bf1.addString(val); + } + for (String val : inputs2) { + bf2.addString(val); + } + + ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); + BloomFilter.serialize(bytesOut, bf1); + byte[] bf1Bytes = bytesOut.toByteArray(); + bytesOut.reset(); + BloomFilter.serialize(bytesOut, bf1); + byte[] bf2Bytes = bytesOut.toByteArray(); + + // Merge bytes + BloomFilter.mergeBloomFilterBytes( + bf1Bytes, 0, bf1Bytes.length, + bf2Bytes, 0, bf2Bytes.length); + + // Deserialize and test + ByteArrayInputStream bytesIn = new ByteArrayInputStream(bf1Bytes, 0, bf1Bytes.length); + BloomFilter bfMerged = BloomFilter.deserialize(bytesIn); + // All values should pass test + for (String val : inputs1) { + bfMerged.addString(val); + } + for (String val : inputs2) { + bfMerged.addString(val); + } + } + + @Test + public void testMergeBloomFilterBytesFailureCases() throws Exception { + BloomFilter bf1 = new BloomFilter(1000); + BloomFilter bf2 = new BloomFilter(200); + // Create bloom filter with same number of bits, but different # hash functions + long[] bits = new long[bf1.getBitSet().length]; + BloomFilter bf3 = new BloomFilter(bits, bf1.getNumHashFunctions() + 1); + + // Serialize to bytes + ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); + BloomFilter.serialize(bytesOut, bf1); + byte[] bf1Bytes = bytesOut.toByteArray(); + + bytesOut.reset(); + BloomFilter.serialize(bytesOut, bf2); + byte[] bf2Bytes = bytesOut.toByteArray(); + + bytesOut.reset(); + BloomFilter.serialize(bytesOut, bf3); + byte[] bf3Bytes = bytesOut.toByteArray(); + + try { + // this should fail + BloomFilter.mergeBloomFilterBytes( + bf1Bytes, 0, bf1Bytes.length, + bf2Bytes, 0, bf2Bytes.length); + Assert.fail("Expected exception not encountered"); + } catch (IllegalArgumentException err) { + // expected + } + + try { + // this should fail + BloomFilter.mergeBloomFilterBytes( + bf1Bytes, 0, bf1Bytes.length, + bf3Bytes, 0, bf3Bytes.length); + Assert.fail("Expected exception not encountered"); + } catch (IllegalArgumentException err) { + // expected + } + } + + @Test + public void testFpp1K() { + int size = 1000; + BloomFilter bf = new BloomFilter(size); + int fp = 0; + for (int i = 0; i < size; i++) { + bf.addLong(i); + } + + for (int i = 0; i < size; i++) { + assertTrue(bf.testLong(i)); + } + + for (int i = 0; i < size; i++) { + int probe = rand.nextInt(); + // out of range probes + if ((probe > size) || (probe < 0)) { + if (bf.testLong(probe)) { + fp++; + } + } + } + + double actualFpp = (double) fp / (double) size; + double expectedFpp = bf.DEFAULT_FPP; + if (actualFpp < expectedFpp) { + assertTrue(actualFpp != 0.0); + } else { + assertEquals(expectedFpp, actualFpp, 0.005); + } + } + + @Test + public void testFpp10K() { + int size = 10_000; + BloomFilter bf = new BloomFilter(size); + int fp = 0; + for (int i = 0; i < size; i++) { + bf.addLong(i); + } + + for (int i = 0; i < size; i++) { + assertTrue(bf.testLong(i)); + } + + for (int i = 0; i < size; i++) { + int probe = rand.nextInt(); + // out of range probes + if ((probe > size) || (probe < 0)) { + if (bf.testLong(probe)) { + fp++; + } + } + } + + double actualFpp = (double) fp / (double) size; + double expectedFpp = bf.DEFAULT_FPP; + if (actualFpp < expectedFpp) { + assertTrue(actualFpp != 0.0); + } else { + assertEquals(expectedFpp, actualFpp, 0.005); + } + } + + @Test + public void testFpp1M() { + int size = 1_000_000; + BloomFilter bf = new BloomFilter(size); + int fp = 0; + for (int i = 0; i < size; i++) { + bf.addLong(i); + } + + for (int i = 0; i < size; i++) { + assertTrue(bf.testLong(i)); + } + + for (int i = 0; i < size; i++) { + int probe = rand.nextInt(); + // out of range probes + if ((probe > size) || (probe < 0)) { + if (bf.testLong(probe)) { + fp++; + } + } + } + + double actualFpp = (double) fp / (double) size; + double expectedFpp = bf.DEFAULT_FPP; + if (actualFpp < expectedFpp) { + assertTrue(actualFpp != 0.0); + } else { + assertEquals(expectedFpp, actualFpp, 0.005); + } + } + + @Test + public void testFpp10M() { + int size = 10_000_000; + BloomFilter bf = new BloomFilter(size); + int fp = 0; + for (int i = 0; i < size; i++) { + bf.addLong(i); + } + + for (int i = 0; i < size; i++) { + assertTrue(bf.testLong(i)); + } + + for (int i = 0; i < size; i++) { + int probe = rand.nextInt(); + // out of range probes + if ((probe > size) || (probe < 0)) { + if (bf.testLong(probe)) { + fp++; + } + } + } + + double actualFpp = (double) fp / (double) size; + double expectedFpp = bf.DEFAULT_FPP; + if (actualFpp < expectedFpp) { + assertTrue(actualFpp != 0.0); + } else { + assertEquals(expectedFpp, actualFpp, 0.005); + } + } +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hive/common/util/TestBloomKFilter.java b/storage-api/hive-storage-api/src/test/org/apache/hive/common/util/TestBloomKFilter.java new file mode 100644 index 0000000000000000000000000000000000000000..159fab2dd0b354ba36427aedc1bb79f5af12ce90 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hive/common/util/TestBloomKFilter.java @@ -0,0 +1,699 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.common.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.Random; + +import org.junit.Assert; +import org.junit.Test; + +/** + * + */ +public class TestBloomKFilter { + private static final int COUNT = 100; + Random rand = new Random(123); + // bloom-1 is known to have higher fpp, to make tests pass give room for another 3% + private final double deltaError = 0.03; + + @Test(expected = IllegalArgumentException.class) + public void testBloomIllegalArg1() { + BloomKFilter bf = new BloomKFilter(0); + } + + @Test + public void testBloomNumBits() { + assertEquals(0, BloomKFilter.optimalNumOfBits(0, 0)); + assertEquals(0, BloomKFilter.optimalNumOfBits(0, 1)); + assertEquals(0, BloomKFilter.optimalNumOfBits(1, 1)); + assertEquals(7, BloomKFilter.optimalNumOfBits(1, 0.03)); + assertEquals(72, BloomKFilter.optimalNumOfBits(10, 0.03)); + assertEquals(729, BloomKFilter.optimalNumOfBits(100, 0.03)); + assertEquals(7298, BloomKFilter.optimalNumOfBits(1000, 0.03)); + assertEquals(72984, BloomKFilter.optimalNumOfBits(10000, 0.03)); + assertEquals(729844, BloomKFilter.optimalNumOfBits(100000, 0.03)); + assertEquals(7298440, BloomKFilter.optimalNumOfBits(1000000, 0.03)); + assertEquals(6235224, BloomKFilter.optimalNumOfBits(1000000, 0.05)); + assertEquals(1870567268, BloomKFilter.optimalNumOfBits(300000000, 0.05)); + assertEquals(1437758756, BloomKFilter.optimalNumOfBits(300000000, 0.1)); + assertEquals(432808512, BloomKFilter.optimalNumOfBits(300000000, 0.5)); + assertEquals(1393332198, BloomKFilter.optimalNumOfBits(3000000000L, 0.8)); + assertEquals(657882327, BloomKFilter.optimalNumOfBits(3000000000L, 0.9)); + assertEquals(0, BloomKFilter.optimalNumOfBits(3000000000L, 1)); + + BloomKFilter bloomKFilter = new BloomKFilter(40); + assertEquals(8, bloomKFilter.getBitSet().length); + assertEquals(bloomKFilter.getNumBits(), bloomKFilter.getBitSize()); + } + + @Test + public void testBloomNumHashFunctions() { + assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(-1, -1)); + assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(0, 0)); + assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(10, 0)); + assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(10, 10)); + assertEquals(7, BloomKFilter.optimalNumOfHashFunctions(10, 100)); + assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(100, 100)); + assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(1000, 100)); + assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(10000, 100)); + assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(100000, 100)); + assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(1000000, 100)); + } + + @Test + public void testBloomKFilterBytes() { + BloomKFilter bf = new BloomKFilter(10000); + byte[] val = new byte[]{1, 2, 3}; + byte[] val1 = new byte[]{1, 2, 3, 4}; + byte[] val2 = new byte[]{1, 2, 3, 4, 5}; + byte[] val3 = new byte[]{1, 2, 3, 4, 5, 6}; + + assertEquals(false, bf.test(val)); + assertEquals(false, bf.test(val1)); + assertEquals(false, bf.test(val2)); + assertEquals(false, bf.test(val3)); + bf.add(val); + assertEquals(true, bf.test(val)); + assertEquals(false, bf.test(val1)); + assertEquals(false, bf.test(val2)); + assertEquals(false, bf.test(val3)); + bf.add(val1); + assertEquals(true, bf.test(val)); + assertEquals(true, bf.test(val1)); + assertEquals(false, bf.test(val2)); + assertEquals(false, bf.test(val3)); + bf.add(val2); + assertEquals(true, bf.test(val)); + assertEquals(true, bf.test(val1)); + assertEquals(true, bf.test(val2)); + assertEquals(false, bf.test(val3)); + bf.add(val3); + assertEquals(true, bf.test(val)); + assertEquals(true, bf.test(val1)); + assertEquals(true, bf.test(val2)); + assertEquals(true, bf.test(val3)); + + byte[] randVal = new byte[COUNT]; + for (int i = 0; i < COUNT; i++) { + rand.nextBytes(randVal); + bf.add(randVal); + } + // last value should be present + assertEquals(true, bf.test(randVal)); + // most likely this value should not exist + randVal[0] = 0; + randVal[1] = 0; + randVal[2] = 0; + randVal[3] = 0; + randVal[4] = 0; + assertEquals(false, bf.test(randVal)); + + assertEquals(7808, bf.sizeInBytes()); + } + + @Test + public void testBloomKFilterByte() { + BloomKFilter bf = new BloomKFilter(10000); + byte val = Byte.MIN_VALUE; + byte val1 = 1; + byte val2 = 2; + byte val3 = Byte.MAX_VALUE; + + assertEquals(false, bf.testLong(val)); + assertEquals(false, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val); + assertEquals(true, bf.testLong(val)); + assertEquals(false, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val1); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val2); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(true, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val3); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(true, bf.testLong(val2)); + assertEquals(true, bf.testLong(val3)); + + byte randVal = 0; + for (int i = 0; i < COUNT; i++) { + randVal = (byte) rand.nextInt(Byte.MAX_VALUE); + bf.addLong(randVal); + } + // last value should be present + assertEquals(true, bf.testLong(randVal)); + // most likely this value should not exist + assertEquals(false, bf.testLong((byte) -120)); + + assertEquals(7808, bf.sizeInBytes()); + } + + @Test + public void testBloomKFilterInt() { + BloomKFilter bf = new BloomKFilter(10000); + int val = Integer.MIN_VALUE; + int val1 = 1; + int val2 = 2; + int val3 = Integer.MAX_VALUE; + + assertEquals(false, bf.testLong(val)); + assertEquals(false, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val); + assertEquals(true, bf.testLong(val)); + assertEquals(false, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val1); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val2); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(true, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val3); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(true, bf.testLong(val2)); + assertEquals(true, bf.testLong(val3)); + + int randVal = 0; + for (int i = 0; i < COUNT; i++) { + randVal = rand.nextInt(); + bf.addLong(randVal); + } + // last value should be present + assertEquals(true, bf.testLong(randVal)); + // most likely this value should not exist + assertEquals(false, bf.testLong(-120)); + + assertEquals(7808, bf.sizeInBytes()); + } + + @Test + public void testBloomKFilterLong() { + BloomKFilter bf = new BloomKFilter(10000); + long val = Long.MIN_VALUE; + long val1 = 1; + long val2 = 2; + long val3 = Long.MAX_VALUE; + + assertEquals(false, bf.testLong(val)); + assertEquals(false, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val); + assertEquals(true, bf.testLong(val)); + assertEquals(false, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val1); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(false, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val2); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(true, bf.testLong(val2)); + assertEquals(false, bf.testLong(val3)); + bf.addLong(val3); + assertEquals(true, bf.testLong(val)); + assertEquals(true, bf.testLong(val1)); + assertEquals(true, bf.testLong(val2)); + assertEquals(true, bf.testLong(val3)); + + long randVal = 0; + for (int i = 0; i < COUNT; i++) { + randVal = rand.nextLong(); + bf.addLong(randVal); + } + // last value should be present + assertEquals(true, bf.testLong(randVal)); + // most likely this value should not exist + assertEquals(false, bf.testLong(-120)); + + assertEquals(7808, bf.sizeInBytes()); + } + + @Test + public void testBloomKFilterFloat() { + BloomKFilter bf = new BloomKFilter(10000); + float val = Float.MIN_VALUE; + float val1 = 1.1f; + float val2 = 2.2f; + float val3 = Float.MAX_VALUE; + + assertEquals(false, bf.testDouble(val)); + assertEquals(false, bf.testDouble(val1)); + assertEquals(false, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val); + assertEquals(true, bf.testDouble(val)); + assertEquals(false, bf.testDouble(val1)); + assertEquals(false, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val1); + assertEquals(true, bf.testDouble(val)); + assertEquals(true, bf.testDouble(val1)); + assertEquals(false, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val2); + assertEquals(true, bf.testDouble(val)); + assertEquals(true, bf.testDouble(val1)); + assertEquals(true, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val3); + assertEquals(true, bf.testDouble(val)); + assertEquals(true, bf.testDouble(val1)); + assertEquals(true, bf.testDouble(val2)); + assertEquals(true, bf.testDouble(val3)); + + float randVal = 0; + for (int i = 0; i < COUNT; i++) { + randVal = rand.nextFloat(); + bf.addDouble(randVal); + } + // last value should be present + assertEquals(true, bf.testDouble(randVal)); + // most likely this value should not exist + assertEquals(false, bf.testDouble(-120.2f)); + + assertEquals(7808, bf.sizeInBytes()); + } + + @Test + public void testBloomKFilterDouble() { + BloomKFilter bf = new BloomKFilter(10000); + double val = Double.MIN_VALUE; + double val1 = 1.1d; + double val2 = 2.2d; + double val3 = Double.MAX_VALUE; + + assertEquals(false, bf.testDouble(val)); + assertEquals(false, bf.testDouble(val1)); + assertEquals(false, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val); + assertEquals(true, bf.testDouble(val)); + assertEquals(false, bf.testDouble(val1)); + assertEquals(false, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val1); + assertEquals(true, bf.testDouble(val)); + assertEquals(true, bf.testDouble(val1)); + assertEquals(false, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val2); + assertEquals(true, bf.testDouble(val)); + assertEquals(true, bf.testDouble(val1)); + assertEquals(true, bf.testDouble(val2)); + assertEquals(false, bf.testDouble(val3)); + bf.addDouble(val3); + assertEquals(true, bf.testDouble(val)); + assertEquals(true, bf.testDouble(val1)); + assertEquals(true, bf.testDouble(val2)); + assertEquals(true, bf.testDouble(val3)); + + double randVal = 0; + for (int i = 0; i < COUNT; i++) { + randVal = rand.nextDouble(); + bf.addDouble(randVal); + } + // last value should be present + assertEquals(true, bf.testDouble(randVal)); + // most likely this value should not exist + assertEquals(false, bf.testDouble(-120.2d)); + + assertEquals(7808, bf.sizeInBytes()); + } + + @Test + public void testBloomKFilterString() { + BloomKFilter bf = new BloomKFilter(100000); + String val = "bloo"; + String val1 = "bloom fil"; + String val2 = "bloom filter"; + String val3 = "cuckoo filter"; + + assertEquals(false, bf.testString(val)); + assertEquals(false, bf.testString(val1)); + assertEquals(false, bf.testString(val2)); + assertEquals(false, bf.testString(val3)); + bf.addString(val); + assertEquals(true, bf.testString(val)); + assertEquals(false, bf.testString(val1)); + assertEquals(false, bf.testString(val2)); + assertEquals(false, bf.testString(val3)); + bf.addString(val1); + assertEquals(true, bf.testString(val)); + assertEquals(true, bf.testString(val1)); + assertEquals(false, bf.testString(val2)); + assertEquals(false, bf.testString(val3)); + bf.addString(val2); + assertEquals(true, bf.testString(val)); + assertEquals(true, bf.testString(val1)); + assertEquals(true, bf.testString(val2)); + assertEquals(false, bf.testString(val3)); + bf.addString(val3); + assertEquals(true, bf.testString(val)); + assertEquals(true, bf.testString(val1)); + assertEquals(true, bf.testString(val2)); + assertEquals(true, bf.testString(val3)); + + long randVal = 0; + for (int i = 0; i < COUNT; i++) { + randVal = rand.nextLong(); + bf.addString(Long.toString(randVal)); + } + // last value should be present + assertEquals(true, bf.testString(Long.toString(randVal))); + // most likely this value should not exist + assertEquals(false, bf.testString(Long.toString(-120))); + + assertEquals(77952, bf.sizeInBytes()); + } + + @Test + public void testMerge() { + BloomKFilter bf = new BloomKFilter(10000); + String val = "bloo"; + String val1 = "bloom fil"; + String val2 = "bloom filter"; + String val3 = "cuckoo filter"; + bf.addString(val); + bf.addString(val1); + bf.addString(val2); + bf.addString(val3); + + BloomKFilter bf2 = new BloomKFilter(10000); + String v = "2_bloo"; + String v1 = "2_bloom fil"; + String v2 = "2_bloom filter"; + String v3 = "2_cuckoo filter"; + bf2.addString(v); + bf2.addString(v1); + bf2.addString(v2); + bf2.addString(v3); + + assertEquals(true, bf.testString(val)); + assertEquals(true, bf.testString(val1)); + assertEquals(true, bf.testString(val2)); + assertEquals(true, bf.testString(val3)); + assertEquals(false, bf.testString(v)); + assertEquals(false, bf.testString(v1)); + assertEquals(false, bf.testString(v2)); + assertEquals(false, bf.testString(v3)); + + bf.merge(bf2); + + assertEquals(true, bf.testString(val)); + assertEquals(true, bf.testString(val1)); + assertEquals(true, bf.testString(val2)); + assertEquals(true, bf.testString(val3)); + assertEquals(true, bf.testString(v)); + assertEquals(true, bf.testString(v1)); + assertEquals(true, bf.testString(v2)); + assertEquals(true, bf.testString(v3)); + } + + @Test + public void testSerialize() throws Exception { + BloomKFilter bf1 = new BloomKFilter(10000); + String[] inputs = { + "bloo", + "bloom fil", + "bloom filter", + "cuckoo filter", + }; + + for (String val : inputs) { + bf1.addString(val); + } + + // Serialize/deserialize + ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); + BloomKFilter.serialize(bytesOut, bf1); + ByteArrayInputStream bytesIn = new ByteArrayInputStream(bytesOut.toByteArray()); + BloomKFilter bf2 = BloomKFilter.deserialize(bytesIn); + + for (String val : inputs) { + assertEquals("Testing bf1 with " + val, true, bf1.testString(val)); + assertEquals("Testing bf2 with " + val, true, bf2.testString(val)); + } + } + + @Test + public void testMergeBloomKFilterBytes() throws Exception { + BloomKFilter bf1 = new BloomKFilter(10000); + BloomKFilter bf2 = new BloomKFilter(10000); + + String[] inputs1 = { + "bloo", + "bloom fil", + "bloom filter", + "cuckoo filter", + }; + + String[] inputs2 = { + "2_bloo", + "2_bloom fil", + "2_bloom filter", + "2_cuckoo filter", + }; + + for (String val : inputs1) { + bf1.addString(val); + } + for (String val : inputs2) { + bf2.addString(val); + } + + ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); + BloomKFilter.serialize(bytesOut, bf1); + byte[] bf1Bytes = bytesOut.toByteArray(); + bytesOut.reset(); + BloomKFilter.serialize(bytesOut, bf1); + byte[] bf2Bytes = bytesOut.toByteArray(); + + // Merge bytes + BloomKFilter.mergeBloomFilterBytes( + bf1Bytes, 0, bf1Bytes.length, + bf2Bytes, 0, bf2Bytes.length); + + // Deserialize and test + ByteArrayInputStream bytesIn = new ByteArrayInputStream(bf1Bytes, 0, bf1Bytes.length); + BloomKFilter bfMerged = BloomKFilter.deserialize(bytesIn); + // All values should pass test + for (String val : inputs1) { + bfMerged.addString(val); + } + for (String val : inputs2) { + bfMerged.addString(val); + } + } + + @Test + public void testMergeBloomKFilterBytesFailureCases() throws Exception { + BloomKFilter bf1 = new BloomKFilter(1000); + BloomKFilter bf2 = new BloomKFilter(200); + // Create bloom filter with same number of bits, but different # hash functions + long[] bits = new long[bf1.getBitSet().length]; + BloomKFilter bf3 = new BloomKFilter(bits, bf1.getNumHashFunctions() + 1); + + // Serialize to bytes + ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); + BloomKFilter.serialize(bytesOut, bf1); + byte[] bf1Bytes = bytesOut.toByteArray(); + + bytesOut.reset(); + BloomKFilter.serialize(bytesOut, bf2); + byte[] bf2Bytes = bytesOut.toByteArray(); + + bytesOut.reset(); + BloomKFilter.serialize(bytesOut, bf3); + byte[] bf3Bytes = bytesOut.toByteArray(); + + try { + // this should fail + BloomKFilter.mergeBloomFilterBytes( + bf1Bytes, 0, bf1Bytes.length, + bf2Bytes, 0, bf2Bytes.length); + Assert.fail("Expected exception not encountered"); + } catch (IllegalArgumentException err) { + // expected + } + + try { + // this should fail + BloomKFilter.mergeBloomFilterBytes( + bf1Bytes, 0, bf1Bytes.length, + bf3Bytes, 0, bf3Bytes.length); + Assert.fail("Expected exception not encountered"); + } catch (IllegalArgumentException err) { + // expected + } + } + + @Test + public void testFpp1K() { + int size = 1000; + BloomKFilter bf = new BloomKFilter(size); + int fp = 0; + for (int i = 0; i < size; i++) { + bf.addLong(i); + } + + for (int i = 0; i < size; i++) { + assertTrue(bf.testLong(i)); + } + + for (int i = 0; i < size; i++) { + int probe = rand.nextInt(); + // out of range probes + if ((probe > size) || (probe < 0)) { + if (bf.testLong(probe)) { + fp++; + } + } + } + + double actualFpp = (double) fp / (double) size; + double expectedFpp = bf.DEFAULT_FPP; + if (actualFpp < expectedFpp) { + assertTrue(actualFpp != 0.0); + } else { + assertEquals(expectedFpp, actualFpp, deltaError); + } + } + + @Test + public void testFpp10K() { + int size = 10_000; + BloomKFilter bf = new BloomKFilter(size); + int fp = 0; + for (int i = 0; i < size; i++) { + bf.addLong(i); + } + + for (int i = 0; i < size; i++) { + assertTrue(bf.testLong(i)); + } + + for (int i = 0; i < size; i++) { + int probe = rand.nextInt(); + // out of range probes + if ((probe > size) || (probe < 0)) { + if (bf.testLong(probe)) { + fp++; + } + } + } + + double actualFpp = (double) fp / (double) size; + double expectedFpp = bf.DEFAULT_FPP; + if (actualFpp < expectedFpp) { + assertTrue(actualFpp != 0.0); + } else { + assertEquals(expectedFpp, actualFpp, deltaError); + } + } + + @Test + public void testFpp1M() { + int size = 1_000_000; + BloomKFilter bf = new BloomKFilter(size); + int fp = 0; + for (int i = 0; i < size; i++) { + bf.addLong(i); + } + + for (int i = 0; i < size; i++) { + assertTrue(bf.testLong(i)); + } + + for (int i = 0; i < size; i++) { + int probe = rand.nextInt(); + // out of range probes + if ((probe > size) || (probe < 0)) { + if (bf.testLong(probe)) { + fp++; + } + } + } + + double actualFpp = (double) fp / (double) size; + double expectedFpp = bf.DEFAULT_FPP; + if (actualFpp < expectedFpp) { + assertTrue(actualFpp != 0.0); + } else { + assertEquals(expectedFpp, actualFpp, deltaError); + } + } + + @Test + public void testFpp10M() { + int size = 10_000_000; + BloomKFilter bf = new BloomKFilter(size); + int fp = 0; + for (int i = 0; i < size; i++) { + bf.addLong(i); + } + + for (int i = 0; i < size; i++) { + assertTrue(bf.testLong(i)); + } + + for (int i = 0; i < size; i++) { + int probe = rand.nextInt(); + // out of range probes + if ((probe > size) || (probe < 0)) { + if (bf.testLong(probe)) { + fp++; + } + } + } + + double actualFpp = (double) fp / (double) size; + double expectedFpp = bf.DEFAULT_FPP; + if (actualFpp < expectedFpp) { + assertTrue(actualFpp != 0.0); + } else { + assertEquals(expectedFpp, actualFpp, deltaError); + } + } +} diff --git a/storage-api/hive-storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java b/storage-api/hive-storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java new file mode 100644 index 0000000000000000000000000000000000000000..391ee425cccbe0f7969c17c8ab76d68f4e6f4de2 --- /dev/null +++ b/storage-api/hive-storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java @@ -0,0 +1,248 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.common.util; + +import static org.junit.Assert.assertEquals; +import org.apache.hive.common.util.Murmur3.IncrementalHash32; + +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; + +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.Random; + +/** + * Tests for Murmur3 variants. + */ +public class TestMurmur3 { + + @Test + public void testHashCodesM3_32_string() { + String key = "test"; + int seed = 123; + HashFunction hf = Hashing.murmur3_32(seed); + int hc1 = hf.hashBytes(key.getBytes()).asInt(); + int hc2 = Murmur3.hash32(key.getBytes(), key.getBytes().length, seed); + assertEquals(hc1, hc2); + + key = "testkey"; + hc1 = hf.hashBytes(key.getBytes()).asInt(); + hc2 = Murmur3.hash32(key.getBytes(), key.getBytes().length, seed); + assertEquals(hc1, hc2); + } + + @Test + public void testHashCodesM3_32_ints() { + int seed = 123; + Random rand = new Random(seed); + HashFunction hf = Hashing.murmur3_32(seed); + for (int i = 0; i < 1000; i++) { + int val = rand.nextInt(); + byte[] data = ByteBuffer.allocate(4).putInt(val).array(); + int hc1 = hf.hashBytes(data).asInt(); + int hc2 = Murmur3.hash32(data, data.length, seed); + assertEquals(hc1, hc2); + } + } + + @Test + public void testHashCodesM3_32_longs() { + int seed = 123; + Random rand = new Random(seed); + HashFunction hf = Hashing.murmur3_32(seed); + for (int i = 0; i < 1000; i++) { + long val = rand.nextLong(); + byte[] data = ByteBuffer.allocate(8).putLong(val).array(); + int hc1 = hf.hashBytes(data).asInt(); + int hc2 = Murmur3.hash32(data, data.length, seed); + assertEquals(hc1, hc2); + } + } + + @Test + public void testHashCodesM3_32_double() { + int seed = 123; + Random rand = new Random(seed); + HashFunction hf = Hashing.murmur3_32(seed); + for (int i = 0; i < 1000; i++) { + double val = rand.nextDouble(); + byte[] data = ByteBuffer.allocate(8).putDouble(val).array(); + int hc1 = hf.hashBytes(data).asInt(); + int hc2 = Murmur3.hash32(data, data.length, seed); + assertEquals(hc1, hc2); + } + } + + @Test + public void testHashCodesM3_128_string() { + String key = "test"; + int seed = 123; + HashFunction hf = Hashing.murmur3_128(seed); + // guava stores the hashcodes in little endian order + ByteBuffer buf = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN); + buf.put(hf.hashBytes(key.getBytes()).asBytes()); + buf.flip(); + long gl1 = buf.getLong(); + long gl2 = buf.getLong(8); + long[] hc = Murmur3.hash128(key.getBytes(), 0, key.getBytes().length, seed); + long m1 = hc[0]; + long m2 = hc[1]; + assertEquals(gl1, m1); + assertEquals(gl2, m2); + + key = "testkey128_testkey128"; + buf = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN); + buf.put(hf.hashBytes(key.getBytes()).asBytes()); + buf.flip(); + gl1 = buf.getLong(); + gl2 = buf.getLong(8); + byte[] keyBytes = key.getBytes(); + hc = Murmur3.hash128(keyBytes, 0, keyBytes.length, seed); + m1 = hc[0]; + m2 = hc[1]; + assertEquals(gl1, m1); + assertEquals(gl2, m2); + + byte[] offsetKeyBytes = new byte[keyBytes.length + 35]; + Arrays.fill(offsetKeyBytes, (byte) -1); + System.arraycopy(keyBytes, 0, offsetKeyBytes, 35, keyBytes.length); + hc = Murmur3.hash128(offsetKeyBytes, 35, keyBytes.length, seed); + assertEquals(gl1, hc[0]); + assertEquals(gl2, hc[1]); + } + + @Test + public void testHashCodeM3_64() { + byte[] origin = ("It was the best of times, it was the worst of times," + + " it was the age of wisdom, it was the age of foolishness," + + " it was the epoch of belief, it was the epoch of incredulity," + + " it was the season of Light, it was the season of Darkness," + + " it was the spring of hope, it was the winter of despair," + + " we had everything before us, we had nothing before us," + + " we were all going direct to Heaven," + + " we were all going direct the other way.").getBytes(); + long hash = Murmur3.hash64(origin, 0, origin.length); + assertEquals(305830725663368540L, hash); + + byte[] originOffset = new byte[origin.length + 150]; + Arrays.fill(originOffset, (byte) 123); + System.arraycopy(origin, 0, originOffset, 150, origin.length); + hash = Murmur3.hash64(originOffset, 150, origin.length); + assertEquals(305830725663368540L, hash); + } + + @Test + public void testHashCodesM3_128_ints() { + int seed = 123; + Random rand = new Random(seed); + HashFunction hf = Hashing.murmur3_128(seed); + for (int i = 0; i < 1000; i++) { + int val = rand.nextInt(); + byte[] data = ByteBuffer.allocate(4).putInt(val).array(); + // guava stores the hashcodes in little endian order + ByteBuffer buf = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN); + buf.put(hf.hashBytes(data).asBytes()); + buf.flip(); + long gl1 = buf.getLong(); + long gl2 = buf.getLong(8); + long[] hc = Murmur3.hash128(data, 0, data.length, seed); + long m1 = hc[0]; + long m2 = hc[1]; + assertEquals(gl1, m1); + assertEquals(gl2, m2); + + byte[] offsetData = new byte[data.length + 50]; + System.arraycopy(data, 0, offsetData, 50, data.length); + hc = Murmur3.hash128(offsetData, 50, data.length, seed); + assertEquals(gl1, hc[0]); + assertEquals(gl2, hc[1]); + } + } + + @Test + public void testHashCodesM3_128_longs() { + int seed = 123; + Random rand = new Random(seed); + HashFunction hf = Hashing.murmur3_128(seed); + for (int i = 0; i < 1000; i++) { + long val = rand.nextLong(); + byte[] data = ByteBuffer.allocate(8).putLong(val).array(); + // guava stores the hashcodes in little endian order + ByteBuffer buf = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN); + buf.put(hf.hashBytes(data).asBytes()); + buf.flip(); + long gl1 = buf.getLong(); + long gl2 = buf.getLong(8); + long[] hc = Murmur3.hash128(data, 0, data.length, seed); + long m1 = hc[0]; + long m2 = hc[1]; + assertEquals(gl1, m1); + assertEquals(gl2, m2); + } + } + + @Test + public void testHashCodesM3_128_double() { + int seed = 123; + Random rand = new Random(seed); + HashFunction hf = Hashing.murmur3_128(seed); + for (int i = 0; i < 1000; i++) { + double val = rand.nextDouble(); + byte[] data = ByteBuffer.allocate(8).putDouble(val).array(); + // guava stores the hashcodes in little endian order + ByteBuffer buf = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN); + buf.put(hf.hashBytes(data).asBytes()); + buf.flip(); + long gl1 = buf.getLong(); + long gl2 = buf.getLong(8); + long[] hc = Murmur3.hash128(data, 0, data.length, seed); + long m1 = hc[0]; + long m2 = hc[1]; + assertEquals(gl1, m1); + assertEquals(gl2, m2); + } + } + + + @Test + public void testIncremental() { + final int seed = 123, arraySize = 1023; + byte[] bytes = new byte[arraySize]; + new Random(seed).nextBytes(bytes); + int expected = Murmur3.hash32(bytes); + Murmur3.IncrementalHash32 same = new IncrementalHash32(), diff = new IncrementalHash32(); + for (int blockSize = 1; blockSize <= arraySize; ++blockSize) { + byte[] block = new byte[blockSize]; + same.start(Murmur3.DEFAULT_SEED); + diff.start(Murmur3.DEFAULT_SEED); + for (int offset = 0; offset < arraySize; offset += blockSize) { + int length = Math.min(arraySize - offset, blockSize); + same.add(bytes, offset, length); + System.arraycopy(bytes, offset, block, 0, length); + diff.add(block, 0, length); + } + assertEquals("Block size " + blockSize, expected, same.end()); + assertEquals("Block size " + blockSize, expected, diff.end()); + } + } +} diff --git a/storage-api/pom.xml b/storage-api/pom.xml index 107fa46cffda5925e28dfbc983396bed25214010..9c538df3dc534f3ea6de220ef78b7d576f292f97 100644 --- a/storage-api/pom.xml +++ b/storage-api/pom.xml @@ -24,10 +24,10 @@ org.apache.hive - hive-storage-api + storage-api 3.0.0-SNAPSHOT - jar - Hive Storage API + pom + Hive Storage and SerDe API 2.6 @@ -38,94 +38,13 @@ 1.7.10 2.15 ${basedir}/checkstyle/ + 3.0.0 - - - - - commons-lang - commons-lang - ${commons-lang.version} - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - provided - - - com.google.guava - guava - - - com.google.code.findbugs - jsr305 - - - commons-logging - commons-logging - - - javax.servlet - servlet-api - - - javax.servlet.jsp - jsp-api - - - org.apache.avro - avro - - - org.apache.httpcomponents - httpcore - - - org.apache.httpcomponents - httpclient - - - org.slf4j - slf4j-log4j12 - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jetty-util - - - - - org.slf4j - slf4j-api - ${slf4j.version} - - - - - com.google.guava - guava - ${guava.version} - test - - - junit - junit - ${junit.version} - test - - - commons-logging - commons-logging - ${commons-logging.version} - test - - - + + + serde-api + hive-storage-api + ${basedir}/src/java ${basedir}/src/test diff --git a/storage-api/serde-api/if/serde.thrift b/storage-api/serde-api/if/serde.thrift new file mode 100644 index 0000000000000000000000000000000000000000..d6003e4706338f73892f05fbb2b52a88a9fedcbe --- /dev/null +++ b/storage-api/serde-api/if/serde.thrift @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace java org.apache.hadoop.hive.serde +namespace php org.apache.hadoop.hive.serde +namespace py org_apache_hadoop_hive_serde +namespace cpp Hive + + // name of serialization scheme. +const string SERIALIZATION_LIB = "serialization.lib" +const string SERIALIZATION_CLASS = "serialization.class" +const string SERIALIZATION_FORMAT = "serialization.format" +const string SERIALIZATION_DDL = "serialization.ddl" +const string SERIALIZATION_NULL_FORMAT = "serialization.null.format" +const string SERIALIZATION_ESCAPE_CRLF = "serialization.escape.crlf" +const string SERIALIZATION_LAST_COLUMN_TAKES_REST = "serialization.last.column.takes.rest" +const string SERIALIZATION_SORT_ORDER = "serialization.sort.order" +const string SERIALIZATION_NULL_SORT_ORDER = "serialization.sort.order.null"; +const string SERIALIZATION_USE_JSON_OBJECTS = "serialization.use.json.object" +const string SERIALIZATION_ENCODING = "serialization.encoding" + +const string FIELD_DELIM = "field.delim" +const string COLLECTION_DELIM = "collection.delim" +const string LINE_DELIM = "line.delim" +const string MAPKEY_DELIM = "mapkey.delim" +const string QUOTE_CHAR = "quote.delim" +const string ESCAPE_CHAR = "escape.delim" +const string HEADER_COUNT = "skip.header.line.count" +const string FOOTER_COUNT = "skip.footer.line.count" + +typedef string PrimitiveType +typedef string CollectionType + +const string VOID_TYPE_NAME = "void"; +const string BOOLEAN_TYPE_NAME = "boolean"; +const string TINYINT_TYPE_NAME = "tinyint"; +const string SMALLINT_TYPE_NAME = "smallint"; +const string INT_TYPE_NAME = "int"; +const string BIGINT_TYPE_NAME = "bigint"; +const string FLOAT_TYPE_NAME = "float"; +const string DOUBLE_TYPE_NAME = "double"; +const string STRING_TYPE_NAME = "string"; +const string CHAR_TYPE_NAME = "char"; +const string VARCHAR_TYPE_NAME = "varchar"; +const string DATE_TYPE_NAME = "date"; +const string DATETIME_TYPE_NAME = "datetime"; +const string TIMESTAMP_TYPE_NAME = "timestamp"; +const string DECIMAL_TYPE_NAME = "decimal"; +const string BINARY_TYPE_NAME = "binary"; +const string INTERVAL_YEAR_MONTH_TYPE_NAME = "interval_year_month"; +const string INTERVAL_DAY_TIME_TYPE_NAME = "interval_day_time"; +const string TIMESTAMPLOCALTZ_TYPE_NAME = "timestamp with local time zone"; + +const string LIST_TYPE_NAME = "array"; +const string MAP_TYPE_NAME = "map"; +const string STRUCT_TYPE_NAME = "struct"; +const string UNION_TYPE_NAME = "uniontype"; + +const string LIST_COLUMNS = "columns"; +const string LIST_COLUMN_TYPES = "columns.types"; + +const string TIMESTAMP_FORMATS = "timestamp.formats"; +const string COLUMN_NAME_DELIMITER = "column.name.delimiter"; + +const set PrimitiveTypes = [ + VOID_TYPE_NAME + BOOLEAN_TYPE_NAME + TINYINT_TYPE_NAME + SMALLINT_TYPE_NAME + INT_TYPE_NAME + BIGINT_TYPE_NAME + FLOAT_TYPE_NAME + DOUBLE_TYPE_NAME + STRING_TYPE_NAME + VARCHAR_TYPE_NAME + CHAR_TYPE_NAME + DATE_TYPE_NAME + DATETIME_TYPE_NAME + TIMESTAMP_TYPE_NAME + INTERVAL_YEAR_MONTH_TYPE_NAME + INTERVAL_DAY_TIME_TYPE_NAME + DECIMAL_TYPE_NAME + BINARY_TYPE_NAME + TIMESTAMPLOCALTZ_TYPE_NAME +], + +const set CollectionTypes = [ LIST_TYPE_NAME MAP_TYPE_NAME ], +const set IntegralTypes = [ TINYINT_TYPE_NAME SMALLINT_TYPE_NAME INT_TYPE_NAME BIGINT_TYPE_NAME ], diff --git a/storage-api/serde-api/if/test/complex.thrift b/storage-api/serde-api/if/test/complex.thrift new file mode 100644 index 0000000000000000000000000000000000000000..2aa9bc07d27b4ccc4dfe24cd2c0440e728d027f9 --- /dev/null +++ b/storage-api/serde-api/if/test/complex.thrift @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace java org.apache.hadoop.hive.serde2.thrift.test + +union PropValueUnion { + 1: optional i32 intValue; + 2: optional i64 longValue; + 3: optional string stringValue; + 4: optional double doubleValue; + 5: optional bool flag; + 6: list lString; + 7: map unionMStringString; +} + +struct IntString { + 1: i32 myint; + 2: string myString; + 3: i32 underscore_int; +} + +struct Complex { + 1: i32 aint; + 2: string aString; + 3: list lint; + 4: list lString; + 5: list lintString; + 6: map mStringString; + 7: map>> attributes; + 8: PropValueUnion unionField1; + 9: PropValueUnion unionField2; + 10: PropValueUnion unionField3; +} + +struct SetIntString { + 1: set sIntString; + 2: string aString; +} diff --git a/storage-api/serde-api/if/test/complexpb.proto b/storage-api/serde-api/if/test/complexpb.proto new file mode 100644 index 0000000000000000000000000000000000000000..8d608ca59623e8e12ff89131c29c4f5a9c8059e3 --- /dev/null +++ b/storage-api/serde-api/if/test/complexpb.proto @@ -0,0 +1,33 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package org.apache.hadoop.hive.serde2.proto.test; + +message IntString { + optional int32 myint = 1; + optional string myString = 2; + optional int32 underscore_int = 3; +}; + +message Complex { + optional int32 aint = 1; + optional string aString = 2; + repeated int32 lint = 3; + repeated string lString = 4; + repeated IntString lintString = 5; +}; diff --git a/storage-api/serde-api/if/test/megastruct.thrift b/storage-api/serde-api/if/test/megastruct.thrift new file mode 100644 index 0000000000000000000000000000000000000000..91eb77ef1811bee1936fce1782b6774ed7812ed4 --- /dev/null +++ b/storage-api/serde-api/if/test/megastruct.thrift @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * MetaStruct intends to stress test Hive's thrift support by having all + * sorts of crazy but valid field types. Please add new fields if you find + * a case that's not handled correctly. + */ + +namespace java org.apache.hadoop.hive.serde2.thrift.test + +enum MyEnum { + LLAMA = 1, + ALPACA = 2 +} + +struct MiniStruct { + 1: optional string my_string, + 2: optional MyEnum my_enum +} + +struct MegaStruct { + 1: optional bool my_bool, + 2: optional byte my_byte, + 3: optional i16 my_16bit_int, + 4: optional i32 my_32bit_int, + 5: optional i64 my_64bit_int, + 6: optional double my_double, + 7: optional string my_string, + 8: optional binary my_binary, + 9: optional map my_string_string_map, + 10: optional map my_string_enum_map, + 11: optional map my_enum_string_map, + 12: optional map my_enum_struct_map, + 13: optional map> my_enum_stringlist_map, + 14: optional map> my_enum_structlist_map, + 15: optional list my_stringlist, + 16: optional list my_structlist, + 17: optional list my_enumlist, + 18: optional set my_stringset, + 19: optional set my_enumset, + 20: optional set my_structset +} diff --git a/storage-api/serde-api/if/test/testthrift.thrift b/storage-api/serde-api/if/test/testthrift.thrift new file mode 100644 index 0000000000000000000000000000000000000000..ddf53af9220a6ac006b0297368eb60230237129f --- /dev/null +++ b/storage-api/serde-api/if/test/testthrift.thrift @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace java org.apache.hadoop.hive.serde.test + + +struct InnerStruct { + 1: i32 field0 +} + +struct ThriftTestObj { + 1: i32 field1, + 2: string field2, + 3: list field3 +} diff --git a/storage-api/serde-api/pom.xml b/storage-api/serde-api/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..3357bc35f800d4750a9a0e82e392f2523241f349 --- /dev/null +++ b/storage-api/serde-api/pom.xml @@ -0,0 +1,231 @@ + + + + 4.0.0 + + org.apache.hive + storage-api + 3.0.0-SNAPSHOT + ../pom.xml + + + org.apache.hive + hive-serde-api + jar + Hive Serde API + + + 0.9.3 + 3.0.0-SNAPSHOT + + 1.0b3 + 3.6.1 + 1.7 + + + + org.apache.hadoop + hadoop-annotations + ${hadoop.version} + + + org.apache.thrift + libthrift + ${libthrift.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + com.google.guava + guava + + + com.google.code.findbugs + jsr305 + + + commons-logging + commons-logging + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + org.apache.avro + avro + + + org.apache.httpcomponents + httpcore + + + org.apache.httpcomponents + httpclient + + + org.slf4j + slf4j-log4j12 + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + + + com.google.code.findbugs + jsr305 + ${jsr305.version} + + + org.apache.hive + hive-storage-api + ${storage-api.version} + + + + + + thriftif + + + + org.apache.maven.plugins + maven-antrun-plugin + + + generate-thrift-sources + generate-sources + + + + + + + + + + + + + + + + + + + + + + run + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-property + + enforce + + + + + thrift.home + + + true + + + + + + + + + + + ${basedir}/src/java + ${basedir}/src/test + + + org.apache.maven.plugins + maven-antrun-plugin + ${maven.antrun.plugin.version} + + + ant-contrib + ant-contrib + ${ant.contrib.version} + + + ant + ant + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + generate-sources + + add-source + + + + src/gen/protobuf/gen-java + src/gen/thrift/gen-javabean + + + + + + + + diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/complex_constants.cpp b/storage-api/serde-api/src/gen/thrift/gen-cpp/complex_constants.cpp new file mode 100644 index 0000000000000000000000000000000000000000..768fc38f9638899566a61e202e6cf3e726dd5f21 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/complex_constants.cpp @@ -0,0 +1,17 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#include "complex_constants.h" + + + +const complexConstants g_complex_constants; + +complexConstants::complexConstants() { +} + + + diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/complex_constants.h b/storage-api/serde-api/src/gen/thrift/gen-cpp/complex_constants.h new file mode 100644 index 0000000000000000000000000000000000000000..6b115dfba72c6ea68df0713dfad48c267f8d197d --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/complex_constants.h @@ -0,0 +1,24 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#ifndef complex_CONSTANTS_H +#define complex_CONSTANTS_H + +#include "complex_types.h" + + + +class complexConstants { + public: + complexConstants(); + +}; + +extern const complexConstants g_complex_constants; + + + +#endif diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/complex_types.cpp b/storage-api/serde-api/src/gen/thrift/gen-cpp/complex_types.cpp new file mode 100644 index 0000000000000000000000000000000000000000..411e1b0a1c0a12be94da6cad9ec1b307d5ed5461 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/complex_types.cpp @@ -0,0 +1,721 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#include "complex_types.h" + +#include + + + +const char* PropValueUnion::ascii_fingerprint = "123CD9D82D5B5054B5054EFD63FC8590"; +const uint8_t PropValueUnion::binary_fingerprint[16] = {0x12,0x3C,0xD9,0xD8,0x2D,0x5B,0x50,0x54,0xB5,0x05,0x4E,0xFD,0x63,0xFC,0x85,0x90}; + +uint32_t PropValueUnion::read(::apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->intValue); + this->__isset.intValue = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->longValue); + this->__isset.longValue = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->stringValue); + this->__isset.stringValue = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 4: + if (ftype == ::apache::thrift::protocol::T_DOUBLE) { + xfer += iprot->readDouble(this->doubleValue); + this->__isset.doubleValue = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 5: + if (ftype == ::apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->flag); + this->__isset.flag = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 6: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->lString.clear(); + uint32_t _size0; + ::apache::thrift::protocol::TType _etype3; + xfer += iprot->readListBegin(_etype3, _size0); + this->lString.resize(_size0); + uint32_t _i4; + for (_i4 = 0; _i4 < _size0; ++_i4) + { + xfer += iprot->readString(this->lString[_i4]); + } + xfer += iprot->readListEnd(); + } + this->__isset.lString = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 7: + if (ftype == ::apache::thrift::protocol::T_MAP) { + { + this->unionMStringString.clear(); + uint32_t _size5; + ::apache::thrift::protocol::TType _ktype6; + ::apache::thrift::protocol::TType _vtype7; + xfer += iprot->readMapBegin(_ktype6, _vtype7, _size5); + uint32_t _i9; + for (_i9 = 0; _i9 < _size5; ++_i9) + { + std::string _key10; + xfer += iprot->readString(_key10); + std::string& _val11 = this->unionMStringString[_key10]; + xfer += iprot->readString(_val11); + } + xfer += iprot->readMapEnd(); + } + this->__isset.unionMStringString = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t PropValueUnion::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("PropValueUnion"); + + if (this->__isset.intValue) { + xfer += oprot->writeFieldBegin("intValue", ::apache::thrift::protocol::T_I32, 1); + xfer += oprot->writeI32(this->intValue); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.longValue) { + xfer += oprot->writeFieldBegin("longValue", ::apache::thrift::protocol::T_I64, 2); + xfer += oprot->writeI64(this->longValue); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.stringValue) { + xfer += oprot->writeFieldBegin("stringValue", ::apache::thrift::protocol::T_STRING, 3); + xfer += oprot->writeString(this->stringValue); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.doubleValue) { + xfer += oprot->writeFieldBegin("doubleValue", ::apache::thrift::protocol::T_DOUBLE, 4); + xfer += oprot->writeDouble(this->doubleValue); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.flag) { + xfer += oprot->writeFieldBegin("flag", ::apache::thrift::protocol::T_BOOL, 5); + xfer += oprot->writeBool(this->flag); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldBegin("lString", ::apache::thrift::protocol::T_LIST, 6); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->lString.size())); + std::vector ::const_iterator _iter12; + for (_iter12 = this->lString.begin(); _iter12 != this->lString.end(); ++_iter12) + { + xfer += oprot->writeString((*_iter12)); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("unionMStringString", ::apache::thrift::protocol::T_MAP, 7); + { + xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRING, static_cast(this->unionMStringString.size())); + std::map ::const_iterator _iter13; + for (_iter13 = this->unionMStringString.begin(); _iter13 != this->unionMStringString.end(); ++_iter13) + { + xfer += oprot->writeString(_iter13->first); + xfer += oprot->writeString(_iter13->second); + } + xfer += oprot->writeMapEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(PropValueUnion &a, PropValueUnion &b) { + using ::std::swap; + swap(a.intValue, b.intValue); + swap(a.longValue, b.longValue); + swap(a.stringValue, b.stringValue); + swap(a.doubleValue, b.doubleValue); + swap(a.flag, b.flag); + swap(a.lString, b.lString); + swap(a.unionMStringString, b.unionMStringString); + swap(a.__isset, b.__isset); +} + +const char* IntString::ascii_fingerprint = "52C6DAB6CF51AF617111F6D3964C6503"; +const uint8_t IntString::binary_fingerprint[16] = {0x52,0xC6,0xDA,0xB6,0xCF,0x51,0xAF,0x61,0x71,0x11,0xF6,0xD3,0x96,0x4C,0x65,0x03}; + +uint32_t IntString::read(::apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->myint); + this->__isset.myint = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->myString); + this->__isset.myString = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->underscore_int); + this->__isset.underscore_int = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t IntString::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("IntString"); + + xfer += oprot->writeFieldBegin("myint", ::apache::thrift::protocol::T_I32, 1); + xfer += oprot->writeI32(this->myint); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("myString", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->myString); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("underscore_int", ::apache::thrift::protocol::T_I32, 3); + xfer += oprot->writeI32(this->underscore_int); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(IntString &a, IntString &b) { + using ::std::swap; + swap(a.myint, b.myint); + swap(a.myString, b.myString); + swap(a.underscore_int, b.underscore_int); + swap(a.__isset, b.__isset); +} + +const char* Complex::ascii_fingerprint = "FFA84FEA7037F5858F2BFEDA73AD679A"; +const uint8_t Complex::binary_fingerprint[16] = {0xFF,0xA8,0x4F,0xEA,0x70,0x37,0xF5,0x85,0x8F,0x2B,0xFE,0xDA,0x73,0xAD,0x67,0x9A}; + +uint32_t Complex::read(::apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->aint); + this->__isset.aint = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->aString); + this->__isset.aString = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->lint.clear(); + uint32_t _size14; + ::apache::thrift::protocol::TType _etype17; + xfer += iprot->readListBegin(_etype17, _size14); + this->lint.resize(_size14); + uint32_t _i18; + for (_i18 = 0; _i18 < _size14; ++_i18) + { + xfer += iprot->readI32(this->lint[_i18]); + } + xfer += iprot->readListEnd(); + } + this->__isset.lint = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 4: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->lString.clear(); + uint32_t _size19; + ::apache::thrift::protocol::TType _etype22; + xfer += iprot->readListBegin(_etype22, _size19); + this->lString.resize(_size19); + uint32_t _i23; + for (_i23 = 0; _i23 < _size19; ++_i23) + { + xfer += iprot->readString(this->lString[_i23]); + } + xfer += iprot->readListEnd(); + } + this->__isset.lString = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 5: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->lintString.clear(); + uint32_t _size24; + ::apache::thrift::protocol::TType _etype27; + xfer += iprot->readListBegin(_etype27, _size24); + this->lintString.resize(_size24); + uint32_t _i28; + for (_i28 = 0; _i28 < _size24; ++_i28) + { + xfer += this->lintString[_i28].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.lintString = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 6: + if (ftype == ::apache::thrift::protocol::T_MAP) { + { + this->mStringString.clear(); + uint32_t _size29; + ::apache::thrift::protocol::TType _ktype30; + ::apache::thrift::protocol::TType _vtype31; + xfer += iprot->readMapBegin(_ktype30, _vtype31, _size29); + uint32_t _i33; + for (_i33 = 0; _i33 < _size29; ++_i33) + { + std::string _key34; + xfer += iprot->readString(_key34); + std::string& _val35 = this->mStringString[_key34]; + xfer += iprot->readString(_val35); + } + xfer += iprot->readMapEnd(); + } + this->__isset.mStringString = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 7: + if (ftype == ::apache::thrift::protocol::T_MAP) { + { + this->attributes.clear(); + uint32_t _size36; + ::apache::thrift::protocol::TType _ktype37; + ::apache::thrift::protocol::TType _vtype38; + xfer += iprot->readMapBegin(_ktype37, _vtype38, _size36); + uint32_t _i40; + for (_i40 = 0; _i40 < _size36; ++_i40) + { + std::string _key41; + xfer += iprot->readString(_key41); + std::map > & _val42 = this->attributes[_key41]; + { + _val42.clear(); + uint32_t _size43; + ::apache::thrift::protocol::TType _ktype44; + ::apache::thrift::protocol::TType _vtype45; + xfer += iprot->readMapBegin(_ktype44, _vtype45, _size43); + uint32_t _i47; + for (_i47 = 0; _i47 < _size43; ++_i47) + { + std::string _key48; + xfer += iprot->readString(_key48); + std::map & _val49 = _val42[_key48]; + { + _val49.clear(); + uint32_t _size50; + ::apache::thrift::protocol::TType _ktype51; + ::apache::thrift::protocol::TType _vtype52; + xfer += iprot->readMapBegin(_ktype51, _vtype52, _size50); + uint32_t _i54; + for (_i54 = 0; _i54 < _size50; ++_i54) + { + std::string _key55; + xfer += iprot->readString(_key55); + PropValueUnion& _val56 = _val49[_key55]; + xfer += _val56.read(iprot); + } + xfer += iprot->readMapEnd(); + } + } + xfer += iprot->readMapEnd(); + } + } + xfer += iprot->readMapEnd(); + } + this->__isset.attributes = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 8: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->unionField1.read(iprot); + this->__isset.unionField1 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 9: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->unionField2.read(iprot); + this->__isset.unionField2 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 10: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->unionField3.read(iprot); + this->__isset.unionField3 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t Complex::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("Complex"); + + xfer += oprot->writeFieldBegin("aint", ::apache::thrift::protocol::T_I32, 1); + xfer += oprot->writeI32(this->aint); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("aString", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->aString); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("lint", ::apache::thrift::protocol::T_LIST, 3); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I32, static_cast(this->lint.size())); + std::vector ::const_iterator _iter57; + for (_iter57 = this->lint.begin(); _iter57 != this->lint.end(); ++_iter57) + { + xfer += oprot->writeI32((*_iter57)); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("lString", ::apache::thrift::protocol::T_LIST, 4); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->lString.size())); + std::vector ::const_iterator _iter58; + for (_iter58 = this->lString.begin(); _iter58 != this->lString.end(); ++_iter58) + { + xfer += oprot->writeString((*_iter58)); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("lintString", ::apache::thrift::protocol::T_LIST, 5); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->lintString.size())); + std::vector ::const_iterator _iter59; + for (_iter59 = this->lintString.begin(); _iter59 != this->lintString.end(); ++_iter59) + { + xfer += (*_iter59).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("mStringString", ::apache::thrift::protocol::T_MAP, 6); + { + xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRING, static_cast(this->mStringString.size())); + std::map ::const_iterator _iter60; + for (_iter60 = this->mStringString.begin(); _iter60 != this->mStringString.end(); ++_iter60) + { + xfer += oprot->writeString(_iter60->first); + xfer += oprot->writeString(_iter60->second); + } + xfer += oprot->writeMapEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("attributes", ::apache::thrift::protocol::T_MAP, 7); + { + xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_MAP, static_cast(this->attributes.size())); + std::map > > ::const_iterator _iter61; + for (_iter61 = this->attributes.begin(); _iter61 != this->attributes.end(); ++_iter61) + { + xfer += oprot->writeString(_iter61->first); + { + xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_MAP, static_cast(_iter61->second.size())); + std::map > ::const_iterator _iter62; + for (_iter62 = _iter61->second.begin(); _iter62 != _iter61->second.end(); ++_iter62) + { + xfer += oprot->writeString(_iter62->first); + { + xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRUCT, static_cast(_iter62->second.size())); + std::map ::const_iterator _iter63; + for (_iter63 = _iter62->second.begin(); _iter63 != _iter62->second.end(); ++_iter63) + { + xfer += oprot->writeString(_iter63->first); + xfer += _iter63->second.write(oprot); + } + xfer += oprot->writeMapEnd(); + } + } + xfer += oprot->writeMapEnd(); + } + } + xfer += oprot->writeMapEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("unionField1", ::apache::thrift::protocol::T_STRUCT, 8); + xfer += this->unionField1.write(oprot); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("unionField2", ::apache::thrift::protocol::T_STRUCT, 9); + xfer += this->unionField2.write(oprot); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("unionField3", ::apache::thrift::protocol::T_STRUCT, 10); + xfer += this->unionField3.write(oprot); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(Complex &a, Complex &b) { + using ::std::swap; + swap(a.aint, b.aint); + swap(a.aString, b.aString); + swap(a.lint, b.lint); + swap(a.lString, b.lString); + swap(a.lintString, b.lintString); + swap(a.mStringString, b.mStringString); + swap(a.attributes, b.attributes); + swap(a.unionField1, b.unionField1); + swap(a.unionField2, b.unionField2); + swap(a.unionField3, b.unionField3); + swap(a.__isset, b.__isset); +} + +const char* SetIntString::ascii_fingerprint = "842B41C940D05DFB16183142A90DFC54"; +const uint8_t SetIntString::binary_fingerprint[16] = {0x84,0x2B,0x41,0xC9,0x40,0xD0,0x5D,0xFB,0x16,0x18,0x31,0x42,0xA9,0x0D,0xFC,0x54}; + +uint32_t SetIntString::read(::apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_SET) { + { + this->sIntString.clear(); + uint32_t _size64; + ::apache::thrift::protocol::TType _etype67; + xfer += iprot->readSetBegin(_etype67, _size64); + uint32_t _i68; + for (_i68 = 0; _i68 < _size64; ++_i68) + { + IntString _elem69; + xfer += _elem69.read(iprot); + this->sIntString.insert(_elem69); + } + xfer += iprot->readSetEnd(); + } + this->__isset.sIntString = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->aString); + this->__isset.aString = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t SetIntString::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("SetIntString"); + + xfer += oprot->writeFieldBegin("sIntString", ::apache::thrift::protocol::T_SET, 1); + { + xfer += oprot->writeSetBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->sIntString.size())); + std::set ::const_iterator _iter70; + for (_iter70 = this->sIntString.begin(); _iter70 != this->sIntString.end(); ++_iter70) + { + xfer += (*_iter70).write(oprot); + } + xfer += oprot->writeSetEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("aString", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->aString); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(SetIntString &a, SetIntString &b) { + using ::std::swap; + swap(a.sIntString, b.sIntString); + swap(a.aString, b.aString); + swap(a.__isset, b.__isset); +} + + diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/complex_types.h b/storage-api/serde-api/src/gen/thrift/gen-cpp/complex_types.h new file mode 100644 index 0000000000000000000000000000000000000000..3f4c76033cb1fdbccad9b3c026d8afa23e87e164 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/complex_types.h @@ -0,0 +1,352 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#ifndef complex_TYPES_H +#define complex_TYPES_H + +#include +#include +#include +#include + + + + + +typedef struct _PropValueUnion__isset { + _PropValueUnion__isset() : intValue(false), longValue(false), stringValue(false), doubleValue(false), flag(false), lString(false), unionMStringString(false) {} + bool intValue; + bool longValue; + bool stringValue; + bool doubleValue; + bool flag; + bool lString; + bool unionMStringString; +} _PropValueUnion__isset; + +class PropValueUnion { + public: + + static const char* ascii_fingerprint; // = "123CD9D82D5B5054B5054EFD63FC8590"; + static const uint8_t binary_fingerprint[16]; // = {0x12,0x3C,0xD9,0xD8,0x2D,0x5B,0x50,0x54,0xB5,0x05,0x4E,0xFD,0x63,0xFC,0x85,0x90}; + + PropValueUnion() : intValue(0), longValue(0), stringValue(), doubleValue(0), flag(0) { + } + + virtual ~PropValueUnion() throw() {} + + int32_t intValue; + int64_t longValue; + std::string stringValue; + double doubleValue; + bool flag; + std::vector lString; + std::map unionMStringString; + + _PropValueUnion__isset __isset; + + void __set_intValue(const int32_t val) { + intValue = val; + __isset.intValue = true; + } + + void __set_longValue(const int64_t val) { + longValue = val; + __isset.longValue = true; + } + + void __set_stringValue(const std::string& val) { + stringValue = val; + __isset.stringValue = true; + } + + void __set_doubleValue(const double val) { + doubleValue = val; + __isset.doubleValue = true; + } + + void __set_flag(const bool val) { + flag = val; + __isset.flag = true; + } + + void __set_lString(const std::vector & val) { + lString = val; + } + + void __set_unionMStringString(const std::map & val) { + unionMStringString = val; + } + + bool operator == (const PropValueUnion & rhs) const + { + if (__isset.intValue != rhs.__isset.intValue) + return false; + else if (__isset.intValue && !(intValue == rhs.intValue)) + return false; + if (__isset.longValue != rhs.__isset.longValue) + return false; + else if (__isset.longValue && !(longValue == rhs.longValue)) + return false; + if (__isset.stringValue != rhs.__isset.stringValue) + return false; + else if (__isset.stringValue && !(stringValue == rhs.stringValue)) + return false; + if (__isset.doubleValue != rhs.__isset.doubleValue) + return false; + else if (__isset.doubleValue && !(doubleValue == rhs.doubleValue)) + return false; + if (__isset.flag != rhs.__isset.flag) + return false; + else if (__isset.flag && !(flag == rhs.flag)) + return false; + if (!(lString == rhs.lString)) + return false; + if (!(unionMStringString == rhs.unionMStringString)) + return false; + return true; + } + bool operator != (const PropValueUnion &rhs) const { + return !(*this == rhs); + } + + bool operator < (const PropValueUnion & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +void swap(PropValueUnion &a, PropValueUnion &b); + +typedef struct _IntString__isset { + _IntString__isset() : myint(false), myString(false), underscore_int(false) {} + bool myint; + bool myString; + bool underscore_int; +} _IntString__isset; + +class IntString { + public: + + static const char* ascii_fingerprint; // = "52C6DAB6CF51AF617111F6D3964C6503"; + static const uint8_t binary_fingerprint[16]; // = {0x52,0xC6,0xDA,0xB6,0xCF,0x51,0xAF,0x61,0x71,0x11,0xF6,0xD3,0x96,0x4C,0x65,0x03}; + + IntString() : myint(0), myString(), underscore_int(0) { + } + + virtual ~IntString() throw() {} + + int32_t myint; + std::string myString; + int32_t underscore_int; + + _IntString__isset __isset; + + void __set_myint(const int32_t val) { + myint = val; + } + + void __set_myString(const std::string& val) { + myString = val; + } + + void __set_underscore_int(const int32_t val) { + underscore_int = val; + } + + bool operator == (const IntString & rhs) const + { + if (!(myint == rhs.myint)) + return false; + if (!(myString == rhs.myString)) + return false; + if (!(underscore_int == rhs.underscore_int)) + return false; + return true; + } + bool operator != (const IntString &rhs) const { + return !(*this == rhs); + } + + bool operator < (const IntString & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +void swap(IntString &a, IntString &b); + +typedef struct _Complex__isset { + _Complex__isset() : aint(false), aString(false), lint(false), lString(false), lintString(false), mStringString(false), attributes(false), unionField1(false), unionField2(false), unionField3(false) {} + bool aint; + bool aString; + bool lint; + bool lString; + bool lintString; + bool mStringString; + bool attributes; + bool unionField1; + bool unionField2; + bool unionField3; +} _Complex__isset; + +class Complex { + public: + + static const char* ascii_fingerprint; // = "FFA84FEA7037F5858F2BFEDA73AD679A"; + static const uint8_t binary_fingerprint[16]; // = {0xFF,0xA8,0x4F,0xEA,0x70,0x37,0xF5,0x85,0x8F,0x2B,0xFE,0xDA,0x73,0xAD,0x67,0x9A}; + + Complex() : aint(0), aString() { + } + + virtual ~Complex() throw() {} + + int32_t aint; + std::string aString; + std::vector lint; + std::vector lString; + std::vector lintString; + std::map mStringString; + std::map > > attributes; + PropValueUnion unionField1; + PropValueUnion unionField2; + PropValueUnion unionField3; + + _Complex__isset __isset; + + void __set_aint(const int32_t val) { + aint = val; + } + + void __set_aString(const std::string& val) { + aString = val; + } + + void __set_lint(const std::vector & val) { + lint = val; + } + + void __set_lString(const std::vector & val) { + lString = val; + } + + void __set_lintString(const std::vector & val) { + lintString = val; + } + + void __set_mStringString(const std::map & val) { + mStringString = val; + } + + void __set_attributes(const std::map > > & val) { + attributes = val; + } + + void __set_unionField1(const PropValueUnion& val) { + unionField1 = val; + } + + void __set_unionField2(const PropValueUnion& val) { + unionField2 = val; + } + + void __set_unionField3(const PropValueUnion& val) { + unionField3 = val; + } + + bool operator == (const Complex & rhs) const + { + if (!(aint == rhs.aint)) + return false; + if (!(aString == rhs.aString)) + return false; + if (!(lint == rhs.lint)) + return false; + if (!(lString == rhs.lString)) + return false; + if (!(lintString == rhs.lintString)) + return false; + if (!(mStringString == rhs.mStringString)) + return false; + if (!(attributes == rhs.attributes)) + return false; + if (!(unionField1 == rhs.unionField1)) + return false; + if (!(unionField2 == rhs.unionField2)) + return false; + if (!(unionField3 == rhs.unionField3)) + return false; + return true; + } + bool operator != (const Complex &rhs) const { + return !(*this == rhs); + } + + bool operator < (const Complex & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +void swap(Complex &a, Complex &b); + +typedef struct _SetIntString__isset { + _SetIntString__isset() : sIntString(false), aString(false) {} + bool sIntString; + bool aString; +} _SetIntString__isset; + +class SetIntString { + public: + + static const char* ascii_fingerprint; // = "842B41C940D05DFB16183142A90DFC54"; + static const uint8_t binary_fingerprint[16]; // = {0x84,0x2B,0x41,0xC9,0x40,0xD0,0x5D,0xFB,0x16,0x18,0x31,0x42,0xA9,0x0D,0xFC,0x54}; + + SetIntString() : aString() { + } + + virtual ~SetIntString() throw() {} + + std::set sIntString; + std::string aString; + + _SetIntString__isset __isset; + + void __set_sIntString(const std::set & val) { + sIntString = val; + } + + void __set_aString(const std::string& val) { + aString = val; + } + + bool operator == (const SetIntString & rhs) const + { + if (!(sIntString == rhs.sIntString)) + return false; + if (!(aString == rhs.aString)) + return false; + return true; + } + bool operator != (const SetIntString &rhs) const { + return !(*this == rhs); + } + + bool operator < (const SetIntString & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +void swap(SetIntString &a, SetIntString &b); + + + +#endif diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/megastruct_constants.cpp b/storage-api/serde-api/src/gen/thrift/gen-cpp/megastruct_constants.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4bd82a1055b9be9054cba55689be505a61ec1389 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/megastruct_constants.cpp @@ -0,0 +1,17 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#include "megastruct_constants.h" + + + +const megastructConstants g_megastruct_constants; + +megastructConstants::megastructConstants() { +} + + + diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/megastruct_constants.h b/storage-api/serde-api/src/gen/thrift/gen-cpp/megastruct_constants.h new file mode 100644 index 0000000000000000000000000000000000000000..9897f8fa0a8d02ceffe116534b28e75c9bfeb520 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/megastruct_constants.h @@ -0,0 +1,24 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#ifndef megastruct_CONSTANTS_H +#define megastruct_CONSTANTS_H + +#include "megastruct_types.h" + + + +class megastructConstants { + public: + megastructConstants(); + +}; + +extern const megastructConstants g_megastruct_constants; + + + +#endif diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/megastruct_types.cpp b/storage-api/serde-api/src/gen/thrift/gen-cpp/megastruct_types.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2d46b7fc542e42a09613e8f3bbacc31b390ea7c5 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/megastruct_types.cpp @@ -0,0 +1,752 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#include "megastruct_types.h" + +#include + + + +int _kMyEnumValues[] = { + MyEnum::LLAMA, + MyEnum::ALPACA +}; +const char* _kMyEnumNames[] = { + "LLAMA", + "ALPACA" +}; +const std::map _MyEnum_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(2, _kMyEnumValues, _kMyEnumNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); + +const char* MiniStruct::ascii_fingerprint = "4ED2B10931906B61ED0B1592EE860A37"; +const uint8_t MiniStruct::binary_fingerprint[16] = {0x4E,0xD2,0xB1,0x09,0x31,0x90,0x6B,0x61,0xED,0x0B,0x15,0x92,0xEE,0x86,0x0A,0x37}; + +uint32_t MiniStruct::read(::apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->my_string); + this->__isset.my_string = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_I32) { + int32_t ecast0; + xfer += iprot->readI32(ecast0); + this->my_enum = (MyEnum::type)ecast0; + this->__isset.my_enum = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t MiniStruct::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("MiniStruct"); + + if (this->__isset.my_string) { + xfer += oprot->writeFieldBegin("my_string", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeString(this->my_string); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_enum) { + xfer += oprot->writeFieldBegin("my_enum", ::apache::thrift::protocol::T_I32, 2); + xfer += oprot->writeI32((int32_t)this->my_enum); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(MiniStruct &a, MiniStruct &b) { + using ::std::swap; + swap(a.my_string, b.my_string); + swap(a.my_enum, b.my_enum); + swap(a.__isset, b.__isset); +} + +const char* MegaStruct::ascii_fingerprint = "9979EEF0CA19988228E64220A3AA9120"; +const uint8_t MegaStruct::binary_fingerprint[16] = {0x99,0x79,0xEE,0xF0,0xCA,0x19,0x98,0x82,0x28,0xE6,0x42,0x20,0xA3,0xAA,0x91,0x20}; + +uint32_t MegaStruct::read(::apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->my_bool); + this->__isset.my_bool = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_BYTE) { + xfer += iprot->readByte(this->my_byte); + this->__isset.my_byte = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_I16) { + xfer += iprot->readI16(this->my_16bit_int); + this->__isset.my_16bit_int = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 4: + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->my_32bit_int); + this->__isset.my_32bit_int = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 5: + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->my_64bit_int); + this->__isset.my_64bit_int = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 6: + if (ftype == ::apache::thrift::protocol::T_DOUBLE) { + xfer += iprot->readDouble(this->my_double); + this->__isset.my_double = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 7: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->my_string); + this->__isset.my_string = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 8: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->my_binary); + this->__isset.my_binary = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 9: + if (ftype == ::apache::thrift::protocol::T_MAP) { + { + this->my_string_string_map.clear(); + uint32_t _size1; + ::apache::thrift::protocol::TType _ktype2; + ::apache::thrift::protocol::TType _vtype3; + xfer += iprot->readMapBegin(_ktype2, _vtype3, _size1); + uint32_t _i5; + for (_i5 = 0; _i5 < _size1; ++_i5) + { + std::string _key6; + xfer += iprot->readString(_key6); + std::string& _val7 = this->my_string_string_map[_key6]; + xfer += iprot->readString(_val7); + } + xfer += iprot->readMapEnd(); + } + this->__isset.my_string_string_map = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 10: + if (ftype == ::apache::thrift::protocol::T_MAP) { + { + this->my_string_enum_map.clear(); + uint32_t _size8; + ::apache::thrift::protocol::TType _ktype9; + ::apache::thrift::protocol::TType _vtype10; + xfer += iprot->readMapBegin(_ktype9, _vtype10, _size8); + uint32_t _i12; + for (_i12 = 0; _i12 < _size8; ++_i12) + { + std::string _key13; + xfer += iprot->readString(_key13); + MyEnum::type& _val14 = this->my_string_enum_map[_key13]; + int32_t ecast15; + xfer += iprot->readI32(ecast15); + _val14 = (MyEnum::type)ecast15; + } + xfer += iprot->readMapEnd(); + } + this->__isset.my_string_enum_map = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 11: + if (ftype == ::apache::thrift::protocol::T_MAP) { + { + this->my_enum_string_map.clear(); + uint32_t _size16; + ::apache::thrift::protocol::TType _ktype17; + ::apache::thrift::protocol::TType _vtype18; + xfer += iprot->readMapBegin(_ktype17, _vtype18, _size16); + uint32_t _i20; + for (_i20 = 0; _i20 < _size16; ++_i20) + { + MyEnum::type _key21; + int32_t ecast23; + xfer += iprot->readI32(ecast23); + _key21 = (MyEnum::type)ecast23; + std::string& _val22 = this->my_enum_string_map[_key21]; + xfer += iprot->readString(_val22); + } + xfer += iprot->readMapEnd(); + } + this->__isset.my_enum_string_map = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 12: + if (ftype == ::apache::thrift::protocol::T_MAP) { + { + this->my_enum_struct_map.clear(); + uint32_t _size24; + ::apache::thrift::protocol::TType _ktype25; + ::apache::thrift::protocol::TType _vtype26; + xfer += iprot->readMapBegin(_ktype25, _vtype26, _size24); + uint32_t _i28; + for (_i28 = 0; _i28 < _size24; ++_i28) + { + MyEnum::type _key29; + int32_t ecast31; + xfer += iprot->readI32(ecast31); + _key29 = (MyEnum::type)ecast31; + MiniStruct& _val30 = this->my_enum_struct_map[_key29]; + xfer += _val30.read(iprot); + } + xfer += iprot->readMapEnd(); + } + this->__isset.my_enum_struct_map = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 13: + if (ftype == ::apache::thrift::protocol::T_MAP) { + { + this->my_enum_stringlist_map.clear(); + uint32_t _size32; + ::apache::thrift::protocol::TType _ktype33; + ::apache::thrift::protocol::TType _vtype34; + xfer += iprot->readMapBegin(_ktype33, _vtype34, _size32); + uint32_t _i36; + for (_i36 = 0; _i36 < _size32; ++_i36) + { + MyEnum::type _key37; + int32_t ecast39; + xfer += iprot->readI32(ecast39); + _key37 = (MyEnum::type)ecast39; + std::vector & _val38 = this->my_enum_stringlist_map[_key37]; + { + _val38.clear(); + uint32_t _size40; + ::apache::thrift::protocol::TType _etype43; + xfer += iprot->readListBegin(_etype43, _size40); + _val38.resize(_size40); + uint32_t _i44; + for (_i44 = 0; _i44 < _size40; ++_i44) + { + xfer += iprot->readString(_val38[_i44]); + } + xfer += iprot->readListEnd(); + } + } + xfer += iprot->readMapEnd(); + } + this->__isset.my_enum_stringlist_map = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 14: + if (ftype == ::apache::thrift::protocol::T_MAP) { + { + this->my_enum_structlist_map.clear(); + uint32_t _size45; + ::apache::thrift::protocol::TType _ktype46; + ::apache::thrift::protocol::TType _vtype47; + xfer += iprot->readMapBegin(_ktype46, _vtype47, _size45); + uint32_t _i49; + for (_i49 = 0; _i49 < _size45; ++_i49) + { + MyEnum::type _key50; + int32_t ecast52; + xfer += iprot->readI32(ecast52); + _key50 = (MyEnum::type)ecast52; + std::vector & _val51 = this->my_enum_structlist_map[_key50]; + { + _val51.clear(); + uint32_t _size53; + ::apache::thrift::protocol::TType _etype56; + xfer += iprot->readListBegin(_etype56, _size53); + _val51.resize(_size53); + uint32_t _i57; + for (_i57 = 0; _i57 < _size53; ++_i57) + { + xfer += _val51[_i57].read(iprot); + } + xfer += iprot->readListEnd(); + } + } + xfer += iprot->readMapEnd(); + } + this->__isset.my_enum_structlist_map = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 15: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->my_stringlist.clear(); + uint32_t _size58; + ::apache::thrift::protocol::TType _etype61; + xfer += iprot->readListBegin(_etype61, _size58); + this->my_stringlist.resize(_size58); + uint32_t _i62; + for (_i62 = 0; _i62 < _size58; ++_i62) + { + xfer += iprot->readString(this->my_stringlist[_i62]); + } + xfer += iprot->readListEnd(); + } + this->__isset.my_stringlist = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 16: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->my_structlist.clear(); + uint32_t _size63; + ::apache::thrift::protocol::TType _etype66; + xfer += iprot->readListBegin(_etype66, _size63); + this->my_structlist.resize(_size63); + uint32_t _i67; + for (_i67 = 0; _i67 < _size63; ++_i67) + { + xfer += this->my_structlist[_i67].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.my_structlist = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 17: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->my_enumlist.clear(); + uint32_t _size68; + ::apache::thrift::protocol::TType _etype71; + xfer += iprot->readListBegin(_etype71, _size68); + this->my_enumlist.resize(_size68); + uint32_t _i72; + for (_i72 = 0; _i72 < _size68; ++_i72) + { + int32_t ecast73; + xfer += iprot->readI32(ecast73); + this->my_enumlist[_i72] = (MyEnum::type)ecast73; + } + xfer += iprot->readListEnd(); + } + this->__isset.my_enumlist = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 18: + if (ftype == ::apache::thrift::protocol::T_SET) { + { + this->my_stringset.clear(); + uint32_t _size74; + ::apache::thrift::protocol::TType _etype77; + xfer += iprot->readSetBegin(_etype77, _size74); + uint32_t _i78; + for (_i78 = 0; _i78 < _size74; ++_i78) + { + std::string _elem79; + xfer += iprot->readString(_elem79); + this->my_stringset.insert(_elem79); + } + xfer += iprot->readSetEnd(); + } + this->__isset.my_stringset = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 19: + if (ftype == ::apache::thrift::protocol::T_SET) { + { + this->my_enumset.clear(); + uint32_t _size80; + ::apache::thrift::protocol::TType _etype83; + xfer += iprot->readSetBegin(_etype83, _size80); + uint32_t _i84; + for (_i84 = 0; _i84 < _size80; ++_i84) + { + MyEnum::type _elem85; + int32_t ecast86; + xfer += iprot->readI32(ecast86); + _elem85 = (MyEnum::type)ecast86; + this->my_enumset.insert(_elem85); + } + xfer += iprot->readSetEnd(); + } + this->__isset.my_enumset = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 20: + if (ftype == ::apache::thrift::protocol::T_SET) { + { + this->my_structset.clear(); + uint32_t _size87; + ::apache::thrift::protocol::TType _etype90; + xfer += iprot->readSetBegin(_etype90, _size87); + uint32_t _i91; + for (_i91 = 0; _i91 < _size87; ++_i91) + { + MiniStruct _elem92; + xfer += _elem92.read(iprot); + this->my_structset.insert(_elem92); + } + xfer += iprot->readSetEnd(); + } + this->__isset.my_structset = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t MegaStruct::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("MegaStruct"); + + if (this->__isset.my_bool) { + xfer += oprot->writeFieldBegin("my_bool", ::apache::thrift::protocol::T_BOOL, 1); + xfer += oprot->writeBool(this->my_bool); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_byte) { + xfer += oprot->writeFieldBegin("my_byte", ::apache::thrift::protocol::T_BYTE, 2); + xfer += oprot->writeByte(this->my_byte); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_16bit_int) { + xfer += oprot->writeFieldBegin("my_16bit_int", ::apache::thrift::protocol::T_I16, 3); + xfer += oprot->writeI16(this->my_16bit_int); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_32bit_int) { + xfer += oprot->writeFieldBegin("my_32bit_int", ::apache::thrift::protocol::T_I32, 4); + xfer += oprot->writeI32(this->my_32bit_int); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_64bit_int) { + xfer += oprot->writeFieldBegin("my_64bit_int", ::apache::thrift::protocol::T_I64, 5); + xfer += oprot->writeI64(this->my_64bit_int); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_double) { + xfer += oprot->writeFieldBegin("my_double", ::apache::thrift::protocol::T_DOUBLE, 6); + xfer += oprot->writeDouble(this->my_double); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_string) { + xfer += oprot->writeFieldBegin("my_string", ::apache::thrift::protocol::T_STRING, 7); + xfer += oprot->writeString(this->my_string); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_binary) { + xfer += oprot->writeFieldBegin("my_binary", ::apache::thrift::protocol::T_STRING, 8); + xfer += oprot->writeBinary(this->my_binary); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_string_string_map) { + xfer += oprot->writeFieldBegin("my_string_string_map", ::apache::thrift::protocol::T_MAP, 9); + { + xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_STRING, static_cast(this->my_string_string_map.size())); + std::map ::const_iterator _iter93; + for (_iter93 = this->my_string_string_map.begin(); _iter93 != this->my_string_string_map.end(); ++_iter93) + { + xfer += oprot->writeString(_iter93->first); + xfer += oprot->writeString(_iter93->second); + } + xfer += oprot->writeMapEnd(); + } + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_string_enum_map) { + xfer += oprot->writeFieldBegin("my_string_enum_map", ::apache::thrift::protocol::T_MAP, 10); + { + xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_I32, static_cast(this->my_string_enum_map.size())); + std::map ::const_iterator _iter94; + for (_iter94 = this->my_string_enum_map.begin(); _iter94 != this->my_string_enum_map.end(); ++_iter94) + { + xfer += oprot->writeString(_iter94->first); + xfer += oprot->writeI32((int32_t)_iter94->second); + } + xfer += oprot->writeMapEnd(); + } + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_enum_string_map) { + xfer += oprot->writeFieldBegin("my_enum_string_map", ::apache::thrift::protocol::T_MAP, 11); + { + xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_I32, ::apache::thrift::protocol::T_STRING, static_cast(this->my_enum_string_map.size())); + std::map ::const_iterator _iter95; + for (_iter95 = this->my_enum_string_map.begin(); _iter95 != this->my_enum_string_map.end(); ++_iter95) + { + xfer += oprot->writeI32((int32_t)_iter95->first); + xfer += oprot->writeString(_iter95->second); + } + xfer += oprot->writeMapEnd(); + } + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_enum_struct_map) { + xfer += oprot->writeFieldBegin("my_enum_struct_map", ::apache::thrift::protocol::T_MAP, 12); + { + xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_I32, ::apache::thrift::protocol::T_STRUCT, static_cast(this->my_enum_struct_map.size())); + std::map ::const_iterator _iter96; + for (_iter96 = this->my_enum_struct_map.begin(); _iter96 != this->my_enum_struct_map.end(); ++_iter96) + { + xfer += oprot->writeI32((int32_t)_iter96->first); + xfer += _iter96->second.write(oprot); + } + xfer += oprot->writeMapEnd(); + } + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_enum_stringlist_map) { + xfer += oprot->writeFieldBegin("my_enum_stringlist_map", ::apache::thrift::protocol::T_MAP, 13); + { + xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_I32, ::apache::thrift::protocol::T_LIST, static_cast(this->my_enum_stringlist_map.size())); + std::map > ::const_iterator _iter97; + for (_iter97 = this->my_enum_stringlist_map.begin(); _iter97 != this->my_enum_stringlist_map.end(); ++_iter97) + { + xfer += oprot->writeI32((int32_t)_iter97->first); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(_iter97->second.size())); + std::vector ::const_iterator _iter98; + for (_iter98 = _iter97->second.begin(); _iter98 != _iter97->second.end(); ++_iter98) + { + xfer += oprot->writeString((*_iter98)); + } + xfer += oprot->writeListEnd(); + } + } + xfer += oprot->writeMapEnd(); + } + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_enum_structlist_map) { + xfer += oprot->writeFieldBegin("my_enum_structlist_map", ::apache::thrift::protocol::T_MAP, 14); + { + xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_I32, ::apache::thrift::protocol::T_LIST, static_cast(this->my_enum_structlist_map.size())); + std::map > ::const_iterator _iter99; + for (_iter99 = this->my_enum_structlist_map.begin(); _iter99 != this->my_enum_structlist_map.end(); ++_iter99) + { + xfer += oprot->writeI32((int32_t)_iter99->first); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(_iter99->second.size())); + std::vector ::const_iterator _iter100; + for (_iter100 = _iter99->second.begin(); _iter100 != _iter99->second.end(); ++_iter100) + { + xfer += (*_iter100).write(oprot); + } + xfer += oprot->writeListEnd(); + } + } + xfer += oprot->writeMapEnd(); + } + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_stringlist) { + xfer += oprot->writeFieldBegin("my_stringlist", ::apache::thrift::protocol::T_LIST, 15); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->my_stringlist.size())); + std::vector ::const_iterator _iter101; + for (_iter101 = this->my_stringlist.begin(); _iter101 != this->my_stringlist.end(); ++_iter101) + { + xfer += oprot->writeString((*_iter101)); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_structlist) { + xfer += oprot->writeFieldBegin("my_structlist", ::apache::thrift::protocol::T_LIST, 16); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->my_structlist.size())); + std::vector ::const_iterator _iter102; + for (_iter102 = this->my_structlist.begin(); _iter102 != this->my_structlist.end(); ++_iter102) + { + xfer += (*_iter102).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_enumlist) { + xfer += oprot->writeFieldBegin("my_enumlist", ::apache::thrift::protocol::T_LIST, 17); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I32, static_cast(this->my_enumlist.size())); + std::vector ::const_iterator _iter103; + for (_iter103 = this->my_enumlist.begin(); _iter103 != this->my_enumlist.end(); ++_iter103) + { + xfer += oprot->writeI32((int32_t)(*_iter103)); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_stringset) { + xfer += oprot->writeFieldBegin("my_stringset", ::apache::thrift::protocol::T_SET, 18); + { + xfer += oprot->writeSetBegin(::apache::thrift::protocol::T_STRING, static_cast(this->my_stringset.size())); + std::set ::const_iterator _iter104; + for (_iter104 = this->my_stringset.begin(); _iter104 != this->my_stringset.end(); ++_iter104) + { + xfer += oprot->writeString((*_iter104)); + } + xfer += oprot->writeSetEnd(); + } + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_enumset) { + xfer += oprot->writeFieldBegin("my_enumset", ::apache::thrift::protocol::T_SET, 19); + { + xfer += oprot->writeSetBegin(::apache::thrift::protocol::T_I32, static_cast(this->my_enumset.size())); + std::set ::const_iterator _iter105; + for (_iter105 = this->my_enumset.begin(); _iter105 != this->my_enumset.end(); ++_iter105) + { + xfer += oprot->writeI32((int32_t)(*_iter105)); + } + xfer += oprot->writeSetEnd(); + } + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.my_structset) { + xfer += oprot->writeFieldBegin("my_structset", ::apache::thrift::protocol::T_SET, 20); + { + xfer += oprot->writeSetBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->my_structset.size())); + std::set ::const_iterator _iter106; + for (_iter106 = this->my_structset.begin(); _iter106 != this->my_structset.end(); ++_iter106) + { + xfer += (*_iter106).write(oprot); + } + xfer += oprot->writeSetEnd(); + } + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(MegaStruct &a, MegaStruct &b) { + using ::std::swap; + swap(a.my_bool, b.my_bool); + swap(a.my_byte, b.my_byte); + swap(a.my_16bit_int, b.my_16bit_int); + swap(a.my_32bit_int, b.my_32bit_int); + swap(a.my_64bit_int, b.my_64bit_int); + swap(a.my_double, b.my_double); + swap(a.my_string, b.my_string); + swap(a.my_binary, b.my_binary); + swap(a.my_string_string_map, b.my_string_string_map); + swap(a.my_string_enum_map, b.my_string_enum_map); + swap(a.my_enum_string_map, b.my_enum_string_map); + swap(a.my_enum_struct_map, b.my_enum_struct_map); + swap(a.my_enum_stringlist_map, b.my_enum_stringlist_map); + swap(a.my_enum_structlist_map, b.my_enum_structlist_map); + swap(a.my_stringlist, b.my_stringlist); + swap(a.my_structlist, b.my_structlist); + swap(a.my_enumlist, b.my_enumlist); + swap(a.my_stringset, b.my_stringset); + swap(a.my_enumset, b.my_enumset); + swap(a.my_structset, b.my_structset); + swap(a.__isset, b.__isset); +} + + diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/megastruct_types.h b/storage-api/serde-api/src/gen/thrift/gen-cpp/megastruct_types.h new file mode 100644 index 0000000000000000000000000000000000000000..b5a8295c4421b8093fec14fd62e237af5fc2b4be --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/megastruct_types.h @@ -0,0 +1,342 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#ifndef megastruct_TYPES_H +#define megastruct_TYPES_H + +#include +#include +#include +#include + + + + + +struct MyEnum { + enum type { + LLAMA = 1, + ALPACA = 2 + }; +}; + +extern const std::map _MyEnum_VALUES_TO_NAMES; + +typedef struct _MiniStruct__isset { + _MiniStruct__isset() : my_string(false), my_enum(false) {} + bool my_string; + bool my_enum; +} _MiniStruct__isset; + +class MiniStruct { + public: + + static const char* ascii_fingerprint; // = "4ED2B10931906B61ED0B1592EE860A37"; + static const uint8_t binary_fingerprint[16]; // = {0x4E,0xD2,0xB1,0x09,0x31,0x90,0x6B,0x61,0xED,0x0B,0x15,0x92,0xEE,0x86,0x0A,0x37}; + + MiniStruct() : my_string(), my_enum((MyEnum::type)0) { + } + + virtual ~MiniStruct() throw() {} + + std::string my_string; + MyEnum::type my_enum; + + _MiniStruct__isset __isset; + + void __set_my_string(const std::string& val) { + my_string = val; + __isset.my_string = true; + } + + void __set_my_enum(const MyEnum::type val) { + my_enum = val; + __isset.my_enum = true; + } + + bool operator == (const MiniStruct & rhs) const + { + if (__isset.my_string != rhs.__isset.my_string) + return false; + else if (__isset.my_string && !(my_string == rhs.my_string)) + return false; + if (__isset.my_enum != rhs.__isset.my_enum) + return false; + else if (__isset.my_enum && !(my_enum == rhs.my_enum)) + return false; + return true; + } + bool operator != (const MiniStruct &rhs) const { + return !(*this == rhs); + } + + bool operator < (const MiniStruct & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +void swap(MiniStruct &a, MiniStruct &b); + +typedef struct _MegaStruct__isset { + _MegaStruct__isset() : my_bool(false), my_byte(false), my_16bit_int(false), my_32bit_int(false), my_64bit_int(false), my_double(false), my_string(false), my_binary(false), my_string_string_map(false), my_string_enum_map(false), my_enum_string_map(false), my_enum_struct_map(false), my_enum_stringlist_map(false), my_enum_structlist_map(false), my_stringlist(false), my_structlist(false), my_enumlist(false), my_stringset(false), my_enumset(false), my_structset(false) {} + bool my_bool; + bool my_byte; + bool my_16bit_int; + bool my_32bit_int; + bool my_64bit_int; + bool my_double; + bool my_string; + bool my_binary; + bool my_string_string_map; + bool my_string_enum_map; + bool my_enum_string_map; + bool my_enum_struct_map; + bool my_enum_stringlist_map; + bool my_enum_structlist_map; + bool my_stringlist; + bool my_structlist; + bool my_enumlist; + bool my_stringset; + bool my_enumset; + bool my_structset; +} _MegaStruct__isset; + +class MegaStruct { + public: + + static const char* ascii_fingerprint; // = "9979EEF0CA19988228E64220A3AA9120"; + static const uint8_t binary_fingerprint[16]; // = {0x99,0x79,0xEE,0xF0,0xCA,0x19,0x98,0x82,0x28,0xE6,0x42,0x20,0xA3,0xAA,0x91,0x20}; + + MegaStruct() : my_bool(0), my_byte(0), my_16bit_int(0), my_32bit_int(0), my_64bit_int(0), my_double(0), my_string(), my_binary() { + } + + virtual ~MegaStruct() throw() {} + + bool my_bool; + int8_t my_byte; + int16_t my_16bit_int; + int32_t my_32bit_int; + int64_t my_64bit_int; + double my_double; + std::string my_string; + std::string my_binary; + std::map my_string_string_map; + std::map my_string_enum_map; + std::map my_enum_string_map; + std::map my_enum_struct_map; + std::map > my_enum_stringlist_map; + std::map > my_enum_structlist_map; + std::vector my_stringlist; + std::vector my_structlist; + std::vector my_enumlist; + std::set my_stringset; + std::set my_enumset; + std::set my_structset; + + _MegaStruct__isset __isset; + + void __set_my_bool(const bool val) { + my_bool = val; + __isset.my_bool = true; + } + + void __set_my_byte(const int8_t val) { + my_byte = val; + __isset.my_byte = true; + } + + void __set_my_16bit_int(const int16_t val) { + my_16bit_int = val; + __isset.my_16bit_int = true; + } + + void __set_my_32bit_int(const int32_t val) { + my_32bit_int = val; + __isset.my_32bit_int = true; + } + + void __set_my_64bit_int(const int64_t val) { + my_64bit_int = val; + __isset.my_64bit_int = true; + } + + void __set_my_double(const double val) { + my_double = val; + __isset.my_double = true; + } + + void __set_my_string(const std::string& val) { + my_string = val; + __isset.my_string = true; + } + + void __set_my_binary(const std::string& val) { + my_binary = val; + __isset.my_binary = true; + } + + void __set_my_string_string_map(const std::map & val) { + my_string_string_map = val; + __isset.my_string_string_map = true; + } + + void __set_my_string_enum_map(const std::map & val) { + my_string_enum_map = val; + __isset.my_string_enum_map = true; + } + + void __set_my_enum_string_map(const std::map & val) { + my_enum_string_map = val; + __isset.my_enum_string_map = true; + } + + void __set_my_enum_struct_map(const std::map & val) { + my_enum_struct_map = val; + __isset.my_enum_struct_map = true; + } + + void __set_my_enum_stringlist_map(const std::map > & val) { + my_enum_stringlist_map = val; + __isset.my_enum_stringlist_map = true; + } + + void __set_my_enum_structlist_map(const std::map > & val) { + my_enum_structlist_map = val; + __isset.my_enum_structlist_map = true; + } + + void __set_my_stringlist(const std::vector & val) { + my_stringlist = val; + __isset.my_stringlist = true; + } + + void __set_my_structlist(const std::vector & val) { + my_structlist = val; + __isset.my_structlist = true; + } + + void __set_my_enumlist(const std::vector & val) { + my_enumlist = val; + __isset.my_enumlist = true; + } + + void __set_my_stringset(const std::set & val) { + my_stringset = val; + __isset.my_stringset = true; + } + + void __set_my_enumset(const std::set & val) { + my_enumset = val; + __isset.my_enumset = true; + } + + void __set_my_structset(const std::set & val) { + my_structset = val; + __isset.my_structset = true; + } + + bool operator == (const MegaStruct & rhs) const + { + if (__isset.my_bool != rhs.__isset.my_bool) + return false; + else if (__isset.my_bool && !(my_bool == rhs.my_bool)) + return false; + if (__isset.my_byte != rhs.__isset.my_byte) + return false; + else if (__isset.my_byte && !(my_byte == rhs.my_byte)) + return false; + if (__isset.my_16bit_int != rhs.__isset.my_16bit_int) + return false; + else if (__isset.my_16bit_int && !(my_16bit_int == rhs.my_16bit_int)) + return false; + if (__isset.my_32bit_int != rhs.__isset.my_32bit_int) + return false; + else if (__isset.my_32bit_int && !(my_32bit_int == rhs.my_32bit_int)) + return false; + if (__isset.my_64bit_int != rhs.__isset.my_64bit_int) + return false; + else if (__isset.my_64bit_int && !(my_64bit_int == rhs.my_64bit_int)) + return false; + if (__isset.my_double != rhs.__isset.my_double) + return false; + else if (__isset.my_double && !(my_double == rhs.my_double)) + return false; + if (__isset.my_string != rhs.__isset.my_string) + return false; + else if (__isset.my_string && !(my_string == rhs.my_string)) + return false; + if (__isset.my_binary != rhs.__isset.my_binary) + return false; + else if (__isset.my_binary && !(my_binary == rhs.my_binary)) + return false; + if (__isset.my_string_string_map != rhs.__isset.my_string_string_map) + return false; + else if (__isset.my_string_string_map && !(my_string_string_map == rhs.my_string_string_map)) + return false; + if (__isset.my_string_enum_map != rhs.__isset.my_string_enum_map) + return false; + else if (__isset.my_string_enum_map && !(my_string_enum_map == rhs.my_string_enum_map)) + return false; + if (__isset.my_enum_string_map != rhs.__isset.my_enum_string_map) + return false; + else if (__isset.my_enum_string_map && !(my_enum_string_map == rhs.my_enum_string_map)) + return false; + if (__isset.my_enum_struct_map != rhs.__isset.my_enum_struct_map) + return false; + else if (__isset.my_enum_struct_map && !(my_enum_struct_map == rhs.my_enum_struct_map)) + return false; + if (__isset.my_enum_stringlist_map != rhs.__isset.my_enum_stringlist_map) + return false; + else if (__isset.my_enum_stringlist_map && !(my_enum_stringlist_map == rhs.my_enum_stringlist_map)) + return false; + if (__isset.my_enum_structlist_map != rhs.__isset.my_enum_structlist_map) + return false; + else if (__isset.my_enum_structlist_map && !(my_enum_structlist_map == rhs.my_enum_structlist_map)) + return false; + if (__isset.my_stringlist != rhs.__isset.my_stringlist) + return false; + else if (__isset.my_stringlist && !(my_stringlist == rhs.my_stringlist)) + return false; + if (__isset.my_structlist != rhs.__isset.my_structlist) + return false; + else if (__isset.my_structlist && !(my_structlist == rhs.my_structlist)) + return false; + if (__isset.my_enumlist != rhs.__isset.my_enumlist) + return false; + else if (__isset.my_enumlist && !(my_enumlist == rhs.my_enumlist)) + return false; + if (__isset.my_stringset != rhs.__isset.my_stringset) + return false; + else if (__isset.my_stringset && !(my_stringset == rhs.my_stringset)) + return false; + if (__isset.my_enumset != rhs.__isset.my_enumset) + return false; + else if (__isset.my_enumset && !(my_enumset == rhs.my_enumset)) + return false; + if (__isset.my_structset != rhs.__isset.my_structset) + return false; + else if (__isset.my_structset && !(my_structset == rhs.my_structset)) + return false; + return true; + } + bool operator != (const MegaStruct &rhs) const { + return !(*this == rhs); + } + + bool operator < (const MegaStruct & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +void swap(MegaStruct &a, MegaStruct &b); + + + +#endif diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/serde_constants.cpp b/storage-api/serde-api/src/gen/thrift/gen-cpp/serde_constants.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a14d8c0219cf5373377f410cb38f0a179a3b1c5d --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/serde_constants.cpp @@ -0,0 +1,137 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#include "serde_constants.h" + +namespace Hive { + +const serdeConstants g_serde_constants; + +serdeConstants::serdeConstants() { + SERIALIZATION_LIB = "serialization.lib"; + + SERIALIZATION_CLASS = "serialization.class"; + + SERIALIZATION_FORMAT = "serialization.format"; + + SERIALIZATION_DDL = "serialization.ddl"; + + SERIALIZATION_NULL_FORMAT = "serialization.null.format"; + + SERIALIZATION_ESCAPE_CRLF = "serialization.escape.crlf"; + + SERIALIZATION_LAST_COLUMN_TAKES_REST = "serialization.last.column.takes.rest"; + + SERIALIZATION_SORT_ORDER = "serialization.sort.order"; + + SERIALIZATION_NULL_SORT_ORDER = "serialization.sort.order.null"; + + SERIALIZATION_USE_JSON_OBJECTS = "serialization.use.json.object"; + + SERIALIZATION_ENCODING = "serialization.encoding"; + + FIELD_DELIM = "field.delim"; + + COLLECTION_DELIM = "collection.delim"; + + LINE_DELIM = "line.delim"; + + MAPKEY_DELIM = "mapkey.delim"; + + QUOTE_CHAR = "quote.delim"; + + ESCAPE_CHAR = "escape.delim"; + + HEADER_COUNT = "skip.header.line.count"; + + FOOTER_COUNT = "skip.footer.line.count"; + + VOID_TYPE_NAME = "void"; + + BOOLEAN_TYPE_NAME = "boolean"; + + TINYINT_TYPE_NAME = "tinyint"; + + SMALLINT_TYPE_NAME = "smallint"; + + INT_TYPE_NAME = "int"; + + BIGINT_TYPE_NAME = "bigint"; + + FLOAT_TYPE_NAME = "float"; + + DOUBLE_TYPE_NAME = "double"; + + STRING_TYPE_NAME = "string"; + + CHAR_TYPE_NAME = "char"; + + VARCHAR_TYPE_NAME = "varchar"; + + DATE_TYPE_NAME = "date"; + + DATETIME_TYPE_NAME = "datetime"; + + TIMESTAMP_TYPE_NAME = "timestamp"; + + DECIMAL_TYPE_NAME = "decimal"; + + BINARY_TYPE_NAME = "binary"; + + INTERVAL_YEAR_MONTH_TYPE_NAME = "interval_year_month"; + + INTERVAL_DAY_TIME_TYPE_NAME = "interval_day_time"; + + TIMESTAMPLOCALTZ_TYPE_NAME = "timestamp with local time zone"; + + LIST_TYPE_NAME = "array"; + + MAP_TYPE_NAME = "map"; + + STRUCT_TYPE_NAME = "struct"; + + UNION_TYPE_NAME = "uniontype"; + + LIST_COLUMNS = "columns"; + + LIST_COLUMN_TYPES = "columns.types"; + + TIMESTAMP_FORMATS = "timestamp.formats"; + + COLUMN_NAME_DELIMITER = "column.name.delimiter"; + + PrimitiveTypes.insert("void"); + PrimitiveTypes.insert("boolean"); + PrimitiveTypes.insert("tinyint"); + PrimitiveTypes.insert("smallint"); + PrimitiveTypes.insert("int"); + PrimitiveTypes.insert("bigint"); + PrimitiveTypes.insert("float"); + PrimitiveTypes.insert("double"); + PrimitiveTypes.insert("string"); + PrimitiveTypes.insert("varchar"); + PrimitiveTypes.insert("char"); + PrimitiveTypes.insert("date"); + PrimitiveTypes.insert("datetime"); + PrimitiveTypes.insert("timestamp"); + PrimitiveTypes.insert("interval_year_month"); + PrimitiveTypes.insert("interval_day_time"); + PrimitiveTypes.insert("decimal"); + PrimitiveTypes.insert("binary"); + PrimitiveTypes.insert("timestamp with local time zone"); + + CollectionTypes.insert("array"); + CollectionTypes.insert("map"); + + IntegralTypes.insert("tinyint"); + IntegralTypes.insert("smallint"); + IntegralTypes.insert("int"); + IntegralTypes.insert("bigint"); + +} + +} // namespace + diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/serde_constants.h b/storage-api/serde-api/src/gen/thrift/gen-cpp/serde_constants.h new file mode 100644 index 0000000000000000000000000000000000000000..48963ec7b123fa91216a76ca10264659756fc564 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/serde_constants.h @@ -0,0 +1,73 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#ifndef serde_CONSTANTS_H +#define serde_CONSTANTS_H + +#include "serde_types.h" + +namespace Hive { + +class serdeConstants { + public: + serdeConstants(); + + std::string SERIALIZATION_LIB; + std::string SERIALIZATION_CLASS; + std::string SERIALIZATION_FORMAT; + std::string SERIALIZATION_DDL; + std::string SERIALIZATION_NULL_FORMAT; + std::string SERIALIZATION_ESCAPE_CRLF; + std::string SERIALIZATION_LAST_COLUMN_TAKES_REST; + std::string SERIALIZATION_SORT_ORDER; + std::string SERIALIZATION_NULL_SORT_ORDER; + std::string SERIALIZATION_USE_JSON_OBJECTS; + std::string SERIALIZATION_ENCODING; + std::string FIELD_DELIM; + std::string COLLECTION_DELIM; + std::string LINE_DELIM; + std::string MAPKEY_DELIM; + std::string QUOTE_CHAR; + std::string ESCAPE_CHAR; + std::string HEADER_COUNT; + std::string FOOTER_COUNT; + std::string VOID_TYPE_NAME; + std::string BOOLEAN_TYPE_NAME; + std::string TINYINT_TYPE_NAME; + std::string SMALLINT_TYPE_NAME; + std::string INT_TYPE_NAME; + std::string BIGINT_TYPE_NAME; + std::string FLOAT_TYPE_NAME; + std::string DOUBLE_TYPE_NAME; + std::string STRING_TYPE_NAME; + std::string CHAR_TYPE_NAME; + std::string VARCHAR_TYPE_NAME; + std::string DATE_TYPE_NAME; + std::string DATETIME_TYPE_NAME; + std::string TIMESTAMP_TYPE_NAME; + std::string DECIMAL_TYPE_NAME; + std::string BINARY_TYPE_NAME; + std::string INTERVAL_YEAR_MONTH_TYPE_NAME; + std::string INTERVAL_DAY_TIME_TYPE_NAME; + std::string TIMESTAMPLOCALTZ_TYPE_NAME; + std::string LIST_TYPE_NAME; + std::string MAP_TYPE_NAME; + std::string STRUCT_TYPE_NAME; + std::string UNION_TYPE_NAME; + std::string LIST_COLUMNS; + std::string LIST_COLUMN_TYPES; + std::string TIMESTAMP_FORMATS; + std::string COLUMN_NAME_DELIMITER; + std::set PrimitiveTypes; + std::set CollectionTypes; + std::set IntegralTypes; +}; + +extern const serdeConstants g_serde_constants; + +} // namespace + +#endif diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/serde_types.cpp b/storage-api/serde-api/src/gen/thrift/gen-cpp/serde_types.cpp new file mode 100644 index 0000000000000000000000000000000000000000..20d1fbdec63667c81977c27ac05b346e20c9004f --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/serde_types.cpp @@ -0,0 +1,13 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#include "serde_types.h" + +#include + +namespace Hive { + +} // namespace diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/serde_types.h b/storage-api/serde-api/src/gen/thrift/gen-cpp/serde_types.h new file mode 100644 index 0000000000000000000000000000000000000000..7fafa7d4420d9f50e901b63d707f67a2024c2e86 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/serde_types.h @@ -0,0 +1,25 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#ifndef serde_TYPES_H +#define serde_TYPES_H + +#include +#include +#include +#include + + + +namespace Hive { + +typedef std::string PrimitiveType; + +typedef std::string CollectionType; + +} // namespace + +#endif diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/testthrift_constants.cpp b/storage-api/serde-api/src/gen/thrift/gen-cpp/testthrift_constants.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3dbbf1b1a15f157bf04b4d87cf9cf992c8594313 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/testthrift_constants.cpp @@ -0,0 +1,17 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#include "testthrift_constants.h" + + + +const testthriftConstants g_testthrift_constants; + +testthriftConstants::testthriftConstants() { +} + + + diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/testthrift_constants.h b/storage-api/serde-api/src/gen/thrift/gen-cpp/testthrift_constants.h new file mode 100644 index 0000000000000000000000000000000000000000..76e5f0c82928df2abc6702520fe2495c45979081 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/testthrift_constants.h @@ -0,0 +1,24 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#ifndef testthrift_CONSTANTS_H +#define testthrift_CONSTANTS_H + +#include "testthrift_types.h" + + + +class testthriftConstants { + public: + testthriftConstants(); + +}; + +extern const testthriftConstants g_testthrift_constants; + + + +#endif diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/testthrift_types.cpp b/storage-api/serde-api/src/gen/thrift/gen-cpp/testthrift_types.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7949f23200c5e61d35ad44eff7bfd6f640a1e4a2 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/testthrift_types.cpp @@ -0,0 +1,183 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#include "testthrift_types.h" + +#include + + + +const char* InnerStruct::ascii_fingerprint = "E86CACEB22240450EDCBEFC3A83970E4"; +const uint8_t InnerStruct::binary_fingerprint[16] = {0xE8,0x6C,0xAC,0xEB,0x22,0x24,0x04,0x50,0xED,0xCB,0xEF,0xC3,0xA8,0x39,0x70,0xE4}; + +uint32_t InnerStruct::read(::apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->field0); + this->__isset.field0 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t InnerStruct::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("InnerStruct"); + + xfer += oprot->writeFieldBegin("field0", ::apache::thrift::protocol::T_I32, 1); + xfer += oprot->writeI32(this->field0); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(InnerStruct &a, InnerStruct &b) { + using ::std::swap; + swap(a.field0, b.field0); + swap(a.__isset, b.__isset); +} + +const char* ThriftTestObj::ascii_fingerprint = "2BA5D8DAACFBBE6599779830A6185706"; +const uint8_t ThriftTestObj::binary_fingerprint[16] = {0x2B,0xA5,0xD8,0xDA,0xAC,0xFB,0xBE,0x65,0x99,0x77,0x98,0x30,0xA6,0x18,0x57,0x06}; + +uint32_t ThriftTestObj::read(::apache::thrift::protocol::TProtocol* iprot) { + + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->field1); + this->__isset.field1 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->field2); + this->__isset.field2 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->field3.clear(); + uint32_t _size0; + ::apache::thrift::protocol::TType _etype3; + xfer += iprot->readListBegin(_etype3, _size0); + this->field3.resize(_size0); + uint32_t _i4; + for (_i4 = 0; _i4 < _size0; ++_i4) + { + xfer += this->field3[_i4].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.field3 = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t ThriftTestObj::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("ThriftTestObj"); + + xfer += oprot->writeFieldBegin("field1", ::apache::thrift::protocol::T_I32, 1); + xfer += oprot->writeI32(this->field1); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("field2", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->field2); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("field3", ::apache::thrift::protocol::T_LIST, 3); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->field3.size())); + std::vector ::const_iterator _iter5; + for (_iter5 = this->field3.begin(); _iter5 != this->field3.end(); ++_iter5) + { + xfer += (*_iter5).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(ThriftTestObj &a, ThriftTestObj &b) { + using ::std::swap; + swap(a.field1, b.field1); + swap(a.field2, b.field2); + swap(a.field3, b.field3); + swap(a.__isset, b.__isset); +} + + diff --git a/storage-api/serde-api/src/gen/thrift/gen-cpp/testthrift_types.h b/storage-api/serde-api/src/gen/thrift/gen-cpp/testthrift_types.h new file mode 100644 index 0000000000000000000000000000000000000000..6c84b9fb3fe17b1529feba939254b811d6457f22 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-cpp/testthrift_types.h @@ -0,0 +1,123 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +#ifndef testthrift_TYPES_H +#define testthrift_TYPES_H + +#include +#include +#include +#include + + + + + +typedef struct _InnerStruct__isset { + _InnerStruct__isset() : field0(false) {} + bool field0; +} _InnerStruct__isset; + +class InnerStruct { + public: + + static const char* ascii_fingerprint; // = "E86CACEB22240450EDCBEFC3A83970E4"; + static const uint8_t binary_fingerprint[16]; // = {0xE8,0x6C,0xAC,0xEB,0x22,0x24,0x04,0x50,0xED,0xCB,0xEF,0xC3,0xA8,0x39,0x70,0xE4}; + + InnerStruct() : field0(0) { + } + + virtual ~InnerStruct() throw() {} + + int32_t field0; + + _InnerStruct__isset __isset; + + void __set_field0(const int32_t val) { + field0 = val; + } + + bool operator == (const InnerStruct & rhs) const + { + if (!(field0 == rhs.field0)) + return false; + return true; + } + bool operator != (const InnerStruct &rhs) const { + return !(*this == rhs); + } + + bool operator < (const InnerStruct & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +void swap(InnerStruct &a, InnerStruct &b); + +typedef struct _ThriftTestObj__isset { + _ThriftTestObj__isset() : field1(false), field2(false), field3(false) {} + bool field1; + bool field2; + bool field3; +} _ThriftTestObj__isset; + +class ThriftTestObj { + public: + + static const char* ascii_fingerprint; // = "2BA5D8DAACFBBE6599779830A6185706"; + static const uint8_t binary_fingerprint[16]; // = {0x2B,0xA5,0xD8,0xDA,0xAC,0xFB,0xBE,0x65,0x99,0x77,0x98,0x30,0xA6,0x18,0x57,0x06}; + + ThriftTestObj() : field1(0), field2() { + } + + virtual ~ThriftTestObj() throw() {} + + int32_t field1; + std::string field2; + std::vector field3; + + _ThriftTestObj__isset __isset; + + void __set_field1(const int32_t val) { + field1 = val; + } + + void __set_field2(const std::string& val) { + field2 = val; + } + + void __set_field3(const std::vector & val) { + field3 = val; + } + + bool operator == (const ThriftTestObj & rhs) const + { + if (!(field1 == rhs.field1)) + return false; + if (!(field2 == rhs.field2)) + return false; + if (!(field3 == rhs.field3)) + return false; + return true; + } + bool operator != (const ThriftTestObj &rhs) const { + return !(*this == rhs); + } + + bool operator < (const ThriftTestObj & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +void swap(ThriftTestObj &a, ThriftTestObj &b); + + + +#endif diff --git a/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java new file mode 100644 index 0000000000000000000000000000000000000000..dcf5f4b9b85b71134c991c03d0f548e689f57d2e --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java @@ -0,0 +1,165 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hive.serde; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class serdeConstants { + + public static final String SERIALIZATION_LIB = "serialization.lib"; + + public static final String SERIALIZATION_CLASS = "serialization.class"; + + public static final String SERIALIZATION_FORMAT = "serialization.format"; + + public static final String SERIALIZATION_DDL = "serialization.ddl"; + + public static final String SERIALIZATION_NULL_FORMAT = "serialization.null.format"; + + public static final String SERIALIZATION_ESCAPE_CRLF = "serialization.escape.crlf"; + + public static final String SERIALIZATION_LAST_COLUMN_TAKES_REST = "serialization.last.column.takes.rest"; + + public static final String SERIALIZATION_SORT_ORDER = "serialization.sort.order"; + + public static final String SERIALIZATION_NULL_SORT_ORDER = "serialization.sort.order.null"; + + public static final String SERIALIZATION_USE_JSON_OBJECTS = "serialization.use.json.object"; + + public static final String SERIALIZATION_ENCODING = "serialization.encoding"; + + public static final String FIELD_DELIM = "field.delim"; + + public static final String COLLECTION_DELIM = "collection.delim"; + + public static final String LINE_DELIM = "line.delim"; + + public static final String MAPKEY_DELIM = "mapkey.delim"; + + public static final String QUOTE_CHAR = "quote.delim"; + + public static final String ESCAPE_CHAR = "escape.delim"; + + public static final String HEADER_COUNT = "skip.header.line.count"; + + public static final String FOOTER_COUNT = "skip.footer.line.count"; + + public static final String VOID_TYPE_NAME = "void"; + + public static final String BOOLEAN_TYPE_NAME = "boolean"; + + public static final String TINYINT_TYPE_NAME = "tinyint"; + + public static final String SMALLINT_TYPE_NAME = "smallint"; + + public static final String INT_TYPE_NAME = "int"; + + public static final String BIGINT_TYPE_NAME = "bigint"; + + public static final String FLOAT_TYPE_NAME = "float"; + + public static final String DOUBLE_TYPE_NAME = "double"; + + public static final String STRING_TYPE_NAME = "string"; + + public static final String CHAR_TYPE_NAME = "char"; + + public static final String VARCHAR_TYPE_NAME = "varchar"; + + public static final String DATE_TYPE_NAME = "date"; + + public static final String DATETIME_TYPE_NAME = "datetime"; + + public static final String TIMESTAMP_TYPE_NAME = "timestamp"; + + public static final String DECIMAL_TYPE_NAME = "decimal"; + + public static final String BINARY_TYPE_NAME = "binary"; + + public static final String INTERVAL_YEAR_MONTH_TYPE_NAME = "interval_year_month"; + + public static final String INTERVAL_DAY_TIME_TYPE_NAME = "interval_day_time"; + + public static final String TIMESTAMPLOCALTZ_TYPE_NAME = "timestamp with local time zone"; + + public static final String LIST_TYPE_NAME = "array"; + + public static final String MAP_TYPE_NAME = "map"; + + public static final String STRUCT_TYPE_NAME = "struct"; + + public static final String UNION_TYPE_NAME = "uniontype"; + + public static final String LIST_COLUMNS = "columns"; + + public static final String LIST_COLUMN_TYPES = "columns.types"; + + public static final String TIMESTAMP_FORMATS = "timestamp.formats"; + + public static final String COLUMN_NAME_DELIMITER = "column.name.delimiter"; + + public static final Set PrimitiveTypes = new HashSet(); + static { + PrimitiveTypes.add("void"); + PrimitiveTypes.add("boolean"); + PrimitiveTypes.add("tinyint"); + PrimitiveTypes.add("smallint"); + PrimitiveTypes.add("int"); + PrimitiveTypes.add("bigint"); + PrimitiveTypes.add("float"); + PrimitiveTypes.add("double"); + PrimitiveTypes.add("string"); + PrimitiveTypes.add("varchar"); + PrimitiveTypes.add("char"); + PrimitiveTypes.add("date"); + PrimitiveTypes.add("datetime"); + PrimitiveTypes.add("timestamp"); + PrimitiveTypes.add("interval_year_month"); + PrimitiveTypes.add("interval_day_time"); + PrimitiveTypes.add("decimal"); + PrimitiveTypes.add("binary"); + PrimitiveTypes.add("timestamp with local time zone"); + } + + public static final Set CollectionTypes = new HashSet(); + static { + CollectionTypes.add("array"); + CollectionTypes.add("map"); + } + + public static final Set IntegralTypes = new HashSet(); + static { + IntegralTypes.add("tinyint"); + IntegralTypes.add("smallint"); + IntegralTypes.add("int"); + IntegralTypes.add("bigint"); + } + +} diff --git a/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/test/InnerStruct.java b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/test/InnerStruct.java new file mode 100644 index 0000000000000000000000000000000000000000..72a28ae68c428d8ed2ead4a48047eb65da8363d5 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/test/InnerStruct.java @@ -0,0 +1,389 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hive.serde.test; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class InnerStruct implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("InnerStruct"); + + private static final org.apache.thrift.protocol.TField FIELD0_FIELD_DESC = new org.apache.thrift.protocol.TField("field0", org.apache.thrift.protocol.TType.I32, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new InnerStructStandardSchemeFactory()); + schemes.put(TupleScheme.class, new InnerStructTupleSchemeFactory()); + } + + private int field0; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + FIELD0((short)1, "field0"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // FIELD0 + return FIELD0; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __FIELD0_ISSET_ID = 0; + private byte __isset_bitfield = 0; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.FIELD0, new org.apache.thrift.meta_data.FieldMetaData("field0", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(InnerStruct.class, metaDataMap); + } + + public InnerStruct() { + } + + public InnerStruct( + int field0) + { + this(); + this.field0 = field0; + setField0IsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public InnerStruct(InnerStruct other) { + __isset_bitfield = other.__isset_bitfield; + this.field0 = other.field0; + } + + public InnerStruct deepCopy() { + return new InnerStruct(this); + } + + @Override + public void clear() { + setField0IsSet(false); + this.field0 = 0; + } + + public int getField0() { + return this.field0; + } + + public void setField0(int field0) { + this.field0 = field0; + setField0IsSet(true); + } + + public void unsetField0() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __FIELD0_ISSET_ID); + } + + /** Returns true if field field0 is set (has been assigned a value) and false otherwise */ + public boolean isSetField0() { + return EncodingUtils.testBit(__isset_bitfield, __FIELD0_ISSET_ID); + } + + public void setField0IsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __FIELD0_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case FIELD0: + if (value == null) { + unsetField0(); + } else { + setField0((Integer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case FIELD0: + return Integer.valueOf(getField0()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case FIELD0: + return isSetField0(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof InnerStruct) + return this.equals((InnerStruct)that); + return false; + } + + public boolean equals(InnerStruct that) { + if (that == null) + return false; + + boolean this_present_field0 = true; + boolean that_present_field0 = true; + if (this_present_field0 || that_present_field0) { + if (!(this_present_field0 && that_present_field0)) + return false; + if (this.field0 != that.field0) + return false; + } + + return true; + } + + @Override + public int hashCode() { + HashCodeBuilder builder = new HashCodeBuilder(); + + boolean present_field0 = true; + builder.append(present_field0); + if (present_field0) + builder.append(field0); + + return builder.toHashCode(); + } + + public int compareTo(InnerStruct other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + InnerStruct typedOther = (InnerStruct)other; + + lastComparison = Boolean.valueOf(isSetField0()).compareTo(typedOther.isSetField0()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetField0()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.field0, typedOther.field0); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("InnerStruct("); + boolean first = true; + + sb.append("field0:"); + sb.append(this.field0); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class InnerStructStandardSchemeFactory implements SchemeFactory { + public InnerStructStandardScheme getScheme() { + return new InnerStructStandardScheme(); + } + } + + private static class InnerStructStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, InnerStruct struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // FIELD0 + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.field0 = iprot.readI32(); + struct.setField0IsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, InnerStruct struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(FIELD0_FIELD_DESC); + oprot.writeI32(struct.field0); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class InnerStructTupleSchemeFactory implements SchemeFactory { + public InnerStructTupleScheme getScheme() { + return new InnerStructTupleScheme(); + } + } + + private static class InnerStructTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, InnerStruct struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetField0()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetField0()) { + oprot.writeI32(struct.field0); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, InnerStruct struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.field0 = iprot.readI32(); + struct.setField0IsSet(true); + } + } + } + +} + diff --git a/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/test/ThriftTestObj.java b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/test/ThriftTestObj.java new file mode 100644 index 0000000000000000000000000000000000000000..1b708ddbcbf48cb3cebc68c64bf67ea34c94c09c --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/test/ThriftTestObj.java @@ -0,0 +1,651 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hive.serde.test; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ThriftTestObj implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ThriftTestObj"); + + private static final org.apache.thrift.protocol.TField FIELD1_FIELD_DESC = new org.apache.thrift.protocol.TField("field1", org.apache.thrift.protocol.TType.I32, (short)1); + private static final org.apache.thrift.protocol.TField FIELD2_FIELD_DESC = new org.apache.thrift.protocol.TField("field2", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField FIELD3_FIELD_DESC = new org.apache.thrift.protocol.TField("field3", org.apache.thrift.protocol.TType.LIST, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new ThriftTestObjStandardSchemeFactory()); + schemes.put(TupleScheme.class, new ThriftTestObjTupleSchemeFactory()); + } + + private int field1; // required + private String field2; // required + private List field3; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + FIELD1((short)1, "field1"), + FIELD2((short)2, "field2"), + FIELD3((short)3, "field3"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // FIELD1 + return FIELD1; + case 2: // FIELD2 + return FIELD2; + case 3: // FIELD3 + return FIELD3; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __FIELD1_ISSET_ID = 0; + private byte __isset_bitfield = 0; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.FIELD1, new org.apache.thrift.meta_data.FieldMetaData("field1", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.FIELD2, new org.apache.thrift.meta_data.FieldMetaData("field2", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.FIELD3, new org.apache.thrift.meta_data.FieldMetaData("field3", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, InnerStruct.class)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ThriftTestObj.class, metaDataMap); + } + + public ThriftTestObj() { + } + + public ThriftTestObj( + int field1, + String field2, + List field3) + { + this(); + this.field1 = field1; + setField1IsSet(true); + this.field2 = field2; + this.field3 = field3; + } + + /** + * Performs a deep copy on other. + */ + public ThriftTestObj(ThriftTestObj other) { + __isset_bitfield = other.__isset_bitfield; + this.field1 = other.field1; + if (other.isSetField2()) { + this.field2 = other.field2; + } + if (other.isSetField3()) { + List __this__field3 = new ArrayList(); + for (InnerStruct other_element : other.field3) { + __this__field3.add(new InnerStruct(other_element)); + } + this.field3 = __this__field3; + } + } + + public ThriftTestObj deepCopy() { + return new ThriftTestObj(this); + } + + @Override + public void clear() { + setField1IsSet(false); + this.field1 = 0; + this.field2 = null; + this.field3 = null; + } + + public int getField1() { + return this.field1; + } + + public void setField1(int field1) { + this.field1 = field1; + setField1IsSet(true); + } + + public void unsetField1() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __FIELD1_ISSET_ID); + } + + /** Returns true if field field1 is set (has been assigned a value) and false otherwise */ + public boolean isSetField1() { + return EncodingUtils.testBit(__isset_bitfield, __FIELD1_ISSET_ID); + } + + public void setField1IsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __FIELD1_ISSET_ID, value); + } + + public String getField2() { + return this.field2; + } + + public void setField2(String field2) { + this.field2 = field2; + } + + public void unsetField2() { + this.field2 = null; + } + + /** Returns true if field field2 is set (has been assigned a value) and false otherwise */ + public boolean isSetField2() { + return this.field2 != null; + } + + public void setField2IsSet(boolean value) { + if (!value) { + this.field2 = null; + } + } + + public int getField3Size() { + return (this.field3 == null) ? 0 : this.field3.size(); + } + + public java.util.Iterator getField3Iterator() { + return (this.field3 == null) ? null : this.field3.iterator(); + } + + public void addToField3(InnerStruct elem) { + if (this.field3 == null) { + this.field3 = new ArrayList(); + } + this.field3.add(elem); + } + + public List getField3() { + return this.field3; + } + + public void setField3(List field3) { + this.field3 = field3; + } + + public void unsetField3() { + this.field3 = null; + } + + /** Returns true if field field3 is set (has been assigned a value) and false otherwise */ + public boolean isSetField3() { + return this.field3 != null; + } + + public void setField3IsSet(boolean value) { + if (!value) { + this.field3 = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case FIELD1: + if (value == null) { + unsetField1(); + } else { + setField1((Integer)value); + } + break; + + case FIELD2: + if (value == null) { + unsetField2(); + } else { + setField2((String)value); + } + break; + + case FIELD3: + if (value == null) { + unsetField3(); + } else { + setField3((List)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case FIELD1: + return Integer.valueOf(getField1()); + + case FIELD2: + return getField2(); + + case FIELD3: + return getField3(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case FIELD1: + return isSetField1(); + case FIELD2: + return isSetField2(); + case FIELD3: + return isSetField3(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof ThriftTestObj) + return this.equals((ThriftTestObj)that); + return false; + } + + public boolean equals(ThriftTestObj that) { + if (that == null) + return false; + + boolean this_present_field1 = true; + boolean that_present_field1 = true; + if (this_present_field1 || that_present_field1) { + if (!(this_present_field1 && that_present_field1)) + return false; + if (this.field1 != that.field1) + return false; + } + + boolean this_present_field2 = true && this.isSetField2(); + boolean that_present_field2 = true && that.isSetField2(); + if (this_present_field2 || that_present_field2) { + if (!(this_present_field2 && that_present_field2)) + return false; + if (!this.field2.equals(that.field2)) + return false; + } + + boolean this_present_field3 = true && this.isSetField3(); + boolean that_present_field3 = true && that.isSetField3(); + if (this_present_field3 || that_present_field3) { + if (!(this_present_field3 && that_present_field3)) + return false; + if (!this.field3.equals(that.field3)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + HashCodeBuilder builder = new HashCodeBuilder(); + + boolean present_field1 = true; + builder.append(present_field1); + if (present_field1) + builder.append(field1); + + boolean present_field2 = true && (isSetField2()); + builder.append(present_field2); + if (present_field2) + builder.append(field2); + + boolean present_field3 = true && (isSetField3()); + builder.append(present_field3); + if (present_field3) + builder.append(field3); + + return builder.toHashCode(); + } + + public int compareTo(ThriftTestObj other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + ThriftTestObj typedOther = (ThriftTestObj)other; + + lastComparison = Boolean.valueOf(isSetField1()).compareTo(typedOther.isSetField1()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetField1()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.field1, typedOther.field1); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetField2()).compareTo(typedOther.isSetField2()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetField2()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.field2, typedOther.field2); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetField3()).compareTo(typedOther.isSetField3()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetField3()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.field3, typedOther.field3); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("ThriftTestObj("); + boolean first = true; + + sb.append("field1:"); + sb.append(this.field1); + first = false; + if (!first) sb.append(", "); + sb.append("field2:"); + if (this.field2 == null) { + sb.append("null"); + } else { + sb.append(this.field2); + } + first = false; + if (!first) sb.append(", "); + sb.append("field3:"); + if (this.field3 == null) { + sb.append("null"); + } else { + sb.append(this.field3); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class ThriftTestObjStandardSchemeFactory implements SchemeFactory { + public ThriftTestObjStandardScheme getScheme() { + return new ThriftTestObjStandardScheme(); + } + } + + private static class ThriftTestObjStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, ThriftTestObj struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // FIELD1 + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.field1 = iprot.readI32(); + struct.setField1IsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // FIELD2 + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.field2 = iprot.readString(); + struct.setField2IsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // FIELD3 + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list0 = iprot.readListBegin(); + struct.field3 = new ArrayList(_list0.size); + for (int _i1 = 0; _i1 < _list0.size; ++_i1) + { + InnerStruct _elem2; // required + _elem2 = new InnerStruct(); + _elem2.read(iprot); + struct.field3.add(_elem2); + } + iprot.readListEnd(); + } + struct.setField3IsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, ThriftTestObj struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(FIELD1_FIELD_DESC); + oprot.writeI32(struct.field1); + oprot.writeFieldEnd(); + if (struct.field2 != null) { + oprot.writeFieldBegin(FIELD2_FIELD_DESC); + oprot.writeString(struct.field2); + oprot.writeFieldEnd(); + } + if (struct.field3 != null) { + oprot.writeFieldBegin(FIELD3_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.field3.size())); + for (InnerStruct _iter3 : struct.field3) + { + _iter3.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class ThriftTestObjTupleSchemeFactory implements SchemeFactory { + public ThriftTestObjTupleScheme getScheme() { + return new ThriftTestObjTupleScheme(); + } + } + + private static class ThriftTestObjTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, ThriftTestObj struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetField1()) { + optionals.set(0); + } + if (struct.isSetField2()) { + optionals.set(1); + } + if (struct.isSetField3()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetField1()) { + oprot.writeI32(struct.field1); + } + if (struct.isSetField2()) { + oprot.writeString(struct.field2); + } + if (struct.isSetField3()) { + { + oprot.writeI32(struct.field3.size()); + for (InnerStruct _iter4 : struct.field3) + { + _iter4.write(oprot); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, ThriftTestObj struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.field1 = iprot.readI32(); + struct.setField1IsSet(true); + } + if (incoming.get(1)) { + struct.field2 = iprot.readString(); + struct.setField2IsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TList _list5 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.field3 = new ArrayList(_list5.size); + for (int _i6 = 0; _i6 < _list5.size; ++_i6) + { + InnerStruct _elem7; // required + _elem7 = new InnerStruct(); + _elem7.read(iprot); + struct.field3.add(_elem7); + } + } + struct.setField3IsSet(true); + } + } + } + +} + diff --git a/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java new file mode 100644 index 0000000000000000000000000000000000000000..07ea8b9dfb15775a0a60c82ad3a7fdacba07783c --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/Complex.java @@ -0,0 +1,1719 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hive.serde2.thrift.test; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Complex implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("Complex"); + + private static final org.apache.thrift.protocol.TField AINT_FIELD_DESC = new org.apache.thrift.protocol.TField("aint", org.apache.thrift.protocol.TType.I32, (short)1); + private static final org.apache.thrift.protocol.TField A_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("aString", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField LINT_FIELD_DESC = new org.apache.thrift.protocol.TField("lint", org.apache.thrift.protocol.TType.LIST, (short)3); + private static final org.apache.thrift.protocol.TField L_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("lString", org.apache.thrift.protocol.TType.LIST, (short)4); + private static final org.apache.thrift.protocol.TField LINT_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("lintString", org.apache.thrift.protocol.TType.LIST, (short)5); + private static final org.apache.thrift.protocol.TField M_STRING_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("mStringString", org.apache.thrift.protocol.TType.MAP, (short)6); + private static final org.apache.thrift.protocol.TField ATTRIBUTES_FIELD_DESC = new org.apache.thrift.protocol.TField("attributes", org.apache.thrift.protocol.TType.MAP, (short)7); + private static final org.apache.thrift.protocol.TField UNION_FIELD1_FIELD_DESC = new org.apache.thrift.protocol.TField("unionField1", org.apache.thrift.protocol.TType.STRUCT, (short)8); + private static final org.apache.thrift.protocol.TField UNION_FIELD2_FIELD_DESC = new org.apache.thrift.protocol.TField("unionField2", org.apache.thrift.protocol.TType.STRUCT, (short)9); + private static final org.apache.thrift.protocol.TField UNION_FIELD3_FIELD_DESC = new org.apache.thrift.protocol.TField("unionField3", org.apache.thrift.protocol.TType.STRUCT, (short)10); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new ComplexStandardSchemeFactory()); + schemes.put(TupleScheme.class, new ComplexTupleSchemeFactory()); + } + + private int aint; // required + private String aString; // required + private List lint; // required + private List lString; // required + private List lintString; // required + private Map mStringString; // required + private Map>> attributes; // required + private PropValueUnion unionField1; // required + private PropValueUnion unionField2; // required + private PropValueUnion unionField3; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + AINT((short)1, "aint"), + A_STRING((short)2, "aString"), + LINT((short)3, "lint"), + L_STRING((short)4, "lString"), + LINT_STRING((short)5, "lintString"), + M_STRING_STRING((short)6, "mStringString"), + ATTRIBUTES((short)7, "attributes"), + UNION_FIELD1((short)8, "unionField1"), + UNION_FIELD2((short)9, "unionField2"), + UNION_FIELD3((short)10, "unionField3"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // AINT + return AINT; + case 2: // A_STRING + return A_STRING; + case 3: // LINT + return LINT; + case 4: // L_STRING + return L_STRING; + case 5: // LINT_STRING + return LINT_STRING; + case 6: // M_STRING_STRING + return M_STRING_STRING; + case 7: // ATTRIBUTES + return ATTRIBUTES; + case 8: // UNION_FIELD1 + return UNION_FIELD1; + case 9: // UNION_FIELD2 + return UNION_FIELD2; + case 10: // UNION_FIELD3 + return UNION_FIELD3; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __AINT_ISSET_ID = 0; + private byte __isset_bitfield = 0; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.AINT, new org.apache.thrift.meta_data.FieldMetaData("aint", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.A_STRING, new org.apache.thrift.meta_data.FieldMetaData("aString", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.LINT, new org.apache.thrift.meta_data.FieldMetaData("lint", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)))); + tmpMap.put(_Fields.L_STRING, new org.apache.thrift.meta_data.FieldMetaData("lString", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + tmpMap.put(_Fields.LINT_STRING, new org.apache.thrift.meta_data.FieldMetaData("lintString", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, IntString.class)))); + tmpMap.put(_Fields.M_STRING_STRING, new org.apache.thrift.meta_data.FieldMetaData("mStringString", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + tmpMap.put(_Fields.ATTRIBUTES, new org.apache.thrift.meta_data.FieldMetaData("attributes", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, PropValueUnion.class)))))); + tmpMap.put(_Fields.UNION_FIELD1, new org.apache.thrift.meta_data.FieldMetaData("unionField1", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, PropValueUnion.class))); + tmpMap.put(_Fields.UNION_FIELD2, new org.apache.thrift.meta_data.FieldMetaData("unionField2", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, PropValueUnion.class))); + tmpMap.put(_Fields.UNION_FIELD3, new org.apache.thrift.meta_data.FieldMetaData("unionField3", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, PropValueUnion.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(Complex.class, metaDataMap); + } + + public Complex() { + } + + public Complex( + int aint, + String aString, + List lint, + List lString, + List lintString, + Map mStringString, + Map>> attributes, + PropValueUnion unionField1, + PropValueUnion unionField2, + PropValueUnion unionField3) + { + this(); + this.aint = aint; + setAintIsSet(true); + this.aString = aString; + this.lint = lint; + this.lString = lString; + this.lintString = lintString; + this.mStringString = mStringString; + this.attributes = attributes; + this.unionField1 = unionField1; + this.unionField2 = unionField2; + this.unionField3 = unionField3; + } + + /** + * Performs a deep copy on other. + */ + public Complex(Complex other) { + __isset_bitfield = other.__isset_bitfield; + this.aint = other.aint; + if (other.isSetAString()) { + this.aString = other.aString; + } + if (other.isSetLint()) { + List __this__lint = new ArrayList(); + for (Integer other_element : other.lint) { + __this__lint.add(other_element); + } + this.lint = __this__lint; + } + if (other.isSetLString()) { + List __this__lString = new ArrayList(); + for (String other_element : other.lString) { + __this__lString.add(other_element); + } + this.lString = __this__lString; + } + if (other.isSetLintString()) { + List __this__lintString = new ArrayList(); + for (IntString other_element : other.lintString) { + __this__lintString.add(new IntString(other_element)); + } + this.lintString = __this__lintString; + } + if (other.isSetMStringString()) { + Map __this__mStringString = new HashMap(); + for (Map.Entry other_element : other.mStringString.entrySet()) { + + String other_element_key = other_element.getKey(); + String other_element_value = other_element.getValue(); + + String __this__mStringString_copy_key = other_element_key; + + String __this__mStringString_copy_value = other_element_value; + + __this__mStringString.put(__this__mStringString_copy_key, __this__mStringString_copy_value); + } + this.mStringString = __this__mStringString; + } + if (other.isSetAttributes()) { + Map>> __this__attributes = new HashMap>>(); + for (Map.Entry>> other_element : other.attributes.entrySet()) { + + String other_element_key = other_element.getKey(); + Map> other_element_value = other_element.getValue(); + + String __this__attributes_copy_key = other_element_key; + + Map> __this__attributes_copy_value = new HashMap>(); + for (Map.Entry> other_element_value_element : other_element_value.entrySet()) { + + String other_element_value_element_key = other_element_value_element.getKey(); + Map other_element_value_element_value = other_element_value_element.getValue(); + + String __this__attributes_copy_value_copy_key = other_element_value_element_key; + + Map __this__attributes_copy_value_copy_value = new HashMap(); + for (Map.Entry other_element_value_element_value_element : other_element_value_element_value.entrySet()) { + + String other_element_value_element_value_element_key = other_element_value_element_value_element.getKey(); + PropValueUnion other_element_value_element_value_element_value = other_element_value_element_value_element.getValue(); + + String __this__attributes_copy_value_copy_value_copy_key = other_element_value_element_value_element_key; + + PropValueUnion __this__attributes_copy_value_copy_value_copy_value = new PropValueUnion(other_element_value_element_value_element_value); + + __this__attributes_copy_value_copy_value.put(__this__attributes_copy_value_copy_value_copy_key, __this__attributes_copy_value_copy_value_copy_value); + } + + __this__attributes_copy_value.put(__this__attributes_copy_value_copy_key, __this__attributes_copy_value_copy_value); + } + + __this__attributes.put(__this__attributes_copy_key, __this__attributes_copy_value); + } + this.attributes = __this__attributes; + } + if (other.isSetUnionField1()) { + this.unionField1 = new PropValueUnion(other.unionField1); + } + if (other.isSetUnionField2()) { + this.unionField2 = new PropValueUnion(other.unionField2); + } + if (other.isSetUnionField3()) { + this.unionField3 = new PropValueUnion(other.unionField3); + } + } + + public Complex deepCopy() { + return new Complex(this); + } + + @Override + public void clear() { + setAintIsSet(false); + this.aint = 0; + this.aString = null; + this.lint = null; + this.lString = null; + this.lintString = null; + this.mStringString = null; + this.attributes = null; + this.unionField1 = null; + this.unionField2 = null; + this.unionField3 = null; + } + + public int getAint() { + return this.aint; + } + + public void setAint(int aint) { + this.aint = aint; + setAintIsSet(true); + } + + public void unsetAint() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __AINT_ISSET_ID); + } + + /** Returns true if field aint is set (has been assigned a value) and false otherwise */ + public boolean isSetAint() { + return EncodingUtils.testBit(__isset_bitfield, __AINT_ISSET_ID); + } + + public void setAintIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __AINT_ISSET_ID, value); + } + + public String getAString() { + return this.aString; + } + + public void setAString(String aString) { + this.aString = aString; + } + + public void unsetAString() { + this.aString = null; + } + + /** Returns true if field aString is set (has been assigned a value) and false otherwise */ + public boolean isSetAString() { + return this.aString != null; + } + + public void setAStringIsSet(boolean value) { + if (!value) { + this.aString = null; + } + } + + public int getLintSize() { + return (this.lint == null) ? 0 : this.lint.size(); + } + + public java.util.Iterator getLintIterator() { + return (this.lint == null) ? null : this.lint.iterator(); + } + + public void addToLint(int elem) { + if (this.lint == null) { + this.lint = new ArrayList(); + } + this.lint.add(elem); + } + + public List getLint() { + return this.lint; + } + + public void setLint(List lint) { + this.lint = lint; + } + + public void unsetLint() { + this.lint = null; + } + + /** Returns true if field lint is set (has been assigned a value) and false otherwise */ + public boolean isSetLint() { + return this.lint != null; + } + + public void setLintIsSet(boolean value) { + if (!value) { + this.lint = null; + } + } + + public int getLStringSize() { + return (this.lString == null) ? 0 : this.lString.size(); + } + + public java.util.Iterator getLStringIterator() { + return (this.lString == null) ? null : this.lString.iterator(); + } + + public void addToLString(String elem) { + if (this.lString == null) { + this.lString = new ArrayList(); + } + this.lString.add(elem); + } + + public List getLString() { + return this.lString; + } + + public void setLString(List lString) { + this.lString = lString; + } + + public void unsetLString() { + this.lString = null; + } + + /** Returns true if field lString is set (has been assigned a value) and false otherwise */ + public boolean isSetLString() { + return this.lString != null; + } + + public void setLStringIsSet(boolean value) { + if (!value) { + this.lString = null; + } + } + + public int getLintStringSize() { + return (this.lintString == null) ? 0 : this.lintString.size(); + } + + public java.util.Iterator getLintStringIterator() { + return (this.lintString == null) ? null : this.lintString.iterator(); + } + + public void addToLintString(IntString elem) { + if (this.lintString == null) { + this.lintString = new ArrayList(); + } + this.lintString.add(elem); + } + + public List getLintString() { + return this.lintString; + } + + public void setLintString(List lintString) { + this.lintString = lintString; + } + + public void unsetLintString() { + this.lintString = null; + } + + /** Returns true if field lintString is set (has been assigned a value) and false otherwise */ + public boolean isSetLintString() { + return this.lintString != null; + } + + public void setLintStringIsSet(boolean value) { + if (!value) { + this.lintString = null; + } + } + + public int getMStringStringSize() { + return (this.mStringString == null) ? 0 : this.mStringString.size(); + } + + public void putToMStringString(String key, String val) { + if (this.mStringString == null) { + this.mStringString = new HashMap(); + } + this.mStringString.put(key, val); + } + + public Map getMStringString() { + return this.mStringString; + } + + public void setMStringString(Map mStringString) { + this.mStringString = mStringString; + } + + public void unsetMStringString() { + this.mStringString = null; + } + + /** Returns true if field mStringString is set (has been assigned a value) and false otherwise */ + public boolean isSetMStringString() { + return this.mStringString != null; + } + + public void setMStringStringIsSet(boolean value) { + if (!value) { + this.mStringString = null; + } + } + + public int getAttributesSize() { + return (this.attributes == null) ? 0 : this.attributes.size(); + } + + public void putToAttributes(String key, Map> val) { + if (this.attributes == null) { + this.attributes = new HashMap>>(); + } + this.attributes.put(key, val); + } + + public Map>> getAttributes() { + return this.attributes; + } + + public void setAttributes(Map>> attributes) { + this.attributes = attributes; + } + + public void unsetAttributes() { + this.attributes = null; + } + + /** Returns true if field attributes is set (has been assigned a value) and false otherwise */ + public boolean isSetAttributes() { + return this.attributes != null; + } + + public void setAttributesIsSet(boolean value) { + if (!value) { + this.attributes = null; + } + } + + public PropValueUnion getUnionField1() { + return this.unionField1; + } + + public void setUnionField1(PropValueUnion unionField1) { + this.unionField1 = unionField1; + } + + public void unsetUnionField1() { + this.unionField1 = null; + } + + /** Returns true if field unionField1 is set (has been assigned a value) and false otherwise */ + public boolean isSetUnionField1() { + return this.unionField1 != null; + } + + public void setUnionField1IsSet(boolean value) { + if (!value) { + this.unionField1 = null; + } + } + + public PropValueUnion getUnionField2() { + return this.unionField2; + } + + public void setUnionField2(PropValueUnion unionField2) { + this.unionField2 = unionField2; + } + + public void unsetUnionField2() { + this.unionField2 = null; + } + + /** Returns true if field unionField2 is set (has been assigned a value) and false otherwise */ + public boolean isSetUnionField2() { + return this.unionField2 != null; + } + + public void setUnionField2IsSet(boolean value) { + if (!value) { + this.unionField2 = null; + } + } + + public PropValueUnion getUnionField3() { + return this.unionField3; + } + + public void setUnionField3(PropValueUnion unionField3) { + this.unionField3 = unionField3; + } + + public void unsetUnionField3() { + this.unionField3 = null; + } + + /** Returns true if field unionField3 is set (has been assigned a value) and false otherwise */ + public boolean isSetUnionField3() { + return this.unionField3 != null; + } + + public void setUnionField3IsSet(boolean value) { + if (!value) { + this.unionField3 = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case AINT: + if (value == null) { + unsetAint(); + } else { + setAint((Integer)value); + } + break; + + case A_STRING: + if (value == null) { + unsetAString(); + } else { + setAString((String)value); + } + break; + + case LINT: + if (value == null) { + unsetLint(); + } else { + setLint((List)value); + } + break; + + case L_STRING: + if (value == null) { + unsetLString(); + } else { + setLString((List)value); + } + break; + + case LINT_STRING: + if (value == null) { + unsetLintString(); + } else { + setLintString((List)value); + } + break; + + case M_STRING_STRING: + if (value == null) { + unsetMStringString(); + } else { + setMStringString((Map)value); + } + break; + + case ATTRIBUTES: + if (value == null) { + unsetAttributes(); + } else { + setAttributes((Map>>)value); + } + break; + + case UNION_FIELD1: + if (value == null) { + unsetUnionField1(); + } else { + setUnionField1((PropValueUnion)value); + } + break; + + case UNION_FIELD2: + if (value == null) { + unsetUnionField2(); + } else { + setUnionField2((PropValueUnion)value); + } + break; + + case UNION_FIELD3: + if (value == null) { + unsetUnionField3(); + } else { + setUnionField3((PropValueUnion)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case AINT: + return Integer.valueOf(getAint()); + + case A_STRING: + return getAString(); + + case LINT: + return getLint(); + + case L_STRING: + return getLString(); + + case LINT_STRING: + return getLintString(); + + case M_STRING_STRING: + return getMStringString(); + + case ATTRIBUTES: + return getAttributes(); + + case UNION_FIELD1: + return getUnionField1(); + + case UNION_FIELD2: + return getUnionField2(); + + case UNION_FIELD3: + return getUnionField3(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case AINT: + return isSetAint(); + case A_STRING: + return isSetAString(); + case LINT: + return isSetLint(); + case L_STRING: + return isSetLString(); + case LINT_STRING: + return isSetLintString(); + case M_STRING_STRING: + return isSetMStringString(); + case ATTRIBUTES: + return isSetAttributes(); + case UNION_FIELD1: + return isSetUnionField1(); + case UNION_FIELD2: + return isSetUnionField2(); + case UNION_FIELD3: + return isSetUnionField3(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof Complex) + return this.equals((Complex)that); + return false; + } + + public boolean equals(Complex that) { + if (that == null) + return false; + + boolean this_present_aint = true; + boolean that_present_aint = true; + if (this_present_aint || that_present_aint) { + if (!(this_present_aint && that_present_aint)) + return false; + if (this.aint != that.aint) + return false; + } + + boolean this_present_aString = true && this.isSetAString(); + boolean that_present_aString = true && that.isSetAString(); + if (this_present_aString || that_present_aString) { + if (!(this_present_aString && that_present_aString)) + return false; + if (!this.aString.equals(that.aString)) + return false; + } + + boolean this_present_lint = true && this.isSetLint(); + boolean that_present_lint = true && that.isSetLint(); + if (this_present_lint || that_present_lint) { + if (!(this_present_lint && that_present_lint)) + return false; + if (!this.lint.equals(that.lint)) + return false; + } + + boolean this_present_lString = true && this.isSetLString(); + boolean that_present_lString = true && that.isSetLString(); + if (this_present_lString || that_present_lString) { + if (!(this_present_lString && that_present_lString)) + return false; + if (!this.lString.equals(that.lString)) + return false; + } + + boolean this_present_lintString = true && this.isSetLintString(); + boolean that_present_lintString = true && that.isSetLintString(); + if (this_present_lintString || that_present_lintString) { + if (!(this_present_lintString && that_present_lintString)) + return false; + if (!this.lintString.equals(that.lintString)) + return false; + } + + boolean this_present_mStringString = true && this.isSetMStringString(); + boolean that_present_mStringString = true && that.isSetMStringString(); + if (this_present_mStringString || that_present_mStringString) { + if (!(this_present_mStringString && that_present_mStringString)) + return false; + if (!this.mStringString.equals(that.mStringString)) + return false; + } + + boolean this_present_attributes = true && this.isSetAttributes(); + boolean that_present_attributes = true && that.isSetAttributes(); + if (this_present_attributes || that_present_attributes) { + if (!(this_present_attributes && that_present_attributes)) + return false; + if (!this.attributes.equals(that.attributes)) + return false; + } + + boolean this_present_unionField1 = true && this.isSetUnionField1(); + boolean that_present_unionField1 = true && that.isSetUnionField1(); + if (this_present_unionField1 || that_present_unionField1) { + if (!(this_present_unionField1 && that_present_unionField1)) + return false; + if (!this.unionField1.equals(that.unionField1)) + return false; + } + + boolean this_present_unionField2 = true && this.isSetUnionField2(); + boolean that_present_unionField2 = true && that.isSetUnionField2(); + if (this_present_unionField2 || that_present_unionField2) { + if (!(this_present_unionField2 && that_present_unionField2)) + return false; + if (!this.unionField2.equals(that.unionField2)) + return false; + } + + boolean this_present_unionField3 = true && this.isSetUnionField3(); + boolean that_present_unionField3 = true && that.isSetUnionField3(); + if (this_present_unionField3 || that_present_unionField3) { + if (!(this_present_unionField3 && that_present_unionField3)) + return false; + if (!this.unionField3.equals(that.unionField3)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + HashCodeBuilder builder = new HashCodeBuilder(); + + boolean present_aint = true; + builder.append(present_aint); + if (present_aint) + builder.append(aint); + + boolean present_aString = true && (isSetAString()); + builder.append(present_aString); + if (present_aString) + builder.append(aString); + + boolean present_lint = true && (isSetLint()); + builder.append(present_lint); + if (present_lint) + builder.append(lint); + + boolean present_lString = true && (isSetLString()); + builder.append(present_lString); + if (present_lString) + builder.append(lString); + + boolean present_lintString = true && (isSetLintString()); + builder.append(present_lintString); + if (present_lintString) + builder.append(lintString); + + boolean present_mStringString = true && (isSetMStringString()); + builder.append(present_mStringString); + if (present_mStringString) + builder.append(mStringString); + + boolean present_attributes = true && (isSetAttributes()); + builder.append(present_attributes); + if (present_attributes) + builder.append(attributes); + + boolean present_unionField1 = true && (isSetUnionField1()); + builder.append(present_unionField1); + if (present_unionField1) + builder.append(unionField1); + + boolean present_unionField2 = true && (isSetUnionField2()); + builder.append(present_unionField2); + if (present_unionField2) + builder.append(unionField2); + + boolean present_unionField3 = true && (isSetUnionField3()); + builder.append(present_unionField3); + if (present_unionField3) + builder.append(unionField3); + + return builder.toHashCode(); + } + + public int compareTo(Complex other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + Complex typedOther = (Complex)other; + + lastComparison = Boolean.valueOf(isSetAint()).compareTo(typedOther.isSetAint()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAint()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.aint, typedOther.aint); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAString()).compareTo(typedOther.isSetAString()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAString()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.aString, typedOther.aString); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetLint()).compareTo(typedOther.isSetLint()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetLint()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.lint, typedOther.lint); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetLString()).compareTo(typedOther.isSetLString()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetLString()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.lString, typedOther.lString); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetLintString()).compareTo(typedOther.isSetLintString()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetLintString()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.lintString, typedOther.lintString); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMStringString()).compareTo(typedOther.isSetMStringString()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMStringString()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.mStringString, typedOther.mStringString); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAttributes()).compareTo(typedOther.isSetAttributes()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAttributes()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.attributes, typedOther.attributes); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetUnionField1()).compareTo(typedOther.isSetUnionField1()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetUnionField1()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.unionField1, typedOther.unionField1); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetUnionField2()).compareTo(typedOther.isSetUnionField2()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetUnionField2()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.unionField2, typedOther.unionField2); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetUnionField3()).compareTo(typedOther.isSetUnionField3()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetUnionField3()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.unionField3, typedOther.unionField3); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("Complex("); + boolean first = true; + + sb.append("aint:"); + sb.append(this.aint); + first = false; + if (!first) sb.append(", "); + sb.append("aString:"); + if (this.aString == null) { + sb.append("null"); + } else { + sb.append(this.aString); + } + first = false; + if (!first) sb.append(", "); + sb.append("lint:"); + if (this.lint == null) { + sb.append("null"); + } else { + sb.append(this.lint); + } + first = false; + if (!first) sb.append(", "); + sb.append("lString:"); + if (this.lString == null) { + sb.append("null"); + } else { + sb.append(this.lString); + } + first = false; + if (!first) sb.append(", "); + sb.append("lintString:"); + if (this.lintString == null) { + sb.append("null"); + } else { + sb.append(this.lintString); + } + first = false; + if (!first) sb.append(", "); + sb.append("mStringString:"); + if (this.mStringString == null) { + sb.append("null"); + } else { + sb.append(this.mStringString); + } + first = false; + if (!first) sb.append(", "); + sb.append("attributes:"); + if (this.attributes == null) { + sb.append("null"); + } else { + sb.append(this.attributes); + } + first = false; + if (!first) sb.append(", "); + sb.append("unionField1:"); + if (this.unionField1 == null) { + sb.append("null"); + } else { + sb.append(this.unionField1); + } + first = false; + if (!first) sb.append(", "); + sb.append("unionField2:"); + if (this.unionField2 == null) { + sb.append("null"); + } else { + sb.append(this.unionField2); + } + first = false; + if (!first) sb.append(", "); + sb.append("unionField3:"); + if (this.unionField3 == null) { + sb.append("null"); + } else { + sb.append(this.unionField3); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class ComplexStandardSchemeFactory implements SchemeFactory { + public ComplexStandardScheme getScheme() { + return new ComplexStandardScheme(); + } + } + + private static class ComplexStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, Complex struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // AINT + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.aint = iprot.readI32(); + struct.setAintIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // A_STRING + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.aString = iprot.readString(); + struct.setAStringIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // LINT + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list18 = iprot.readListBegin(); + struct.lint = new ArrayList(_list18.size); + for (int _i19 = 0; _i19 < _list18.size; ++_i19) + { + int _elem20; // required + _elem20 = iprot.readI32(); + struct.lint.add(_elem20); + } + iprot.readListEnd(); + } + struct.setLintIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // L_STRING + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list21 = iprot.readListBegin(); + struct.lString = new ArrayList(_list21.size); + for (int _i22 = 0; _i22 < _list21.size; ++_i22) + { + String _elem23; // required + _elem23 = iprot.readString(); + struct.lString.add(_elem23); + } + iprot.readListEnd(); + } + struct.setLStringIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // LINT_STRING + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list24 = iprot.readListBegin(); + struct.lintString = new ArrayList(_list24.size); + for (int _i25 = 0; _i25 < _list24.size; ++_i25) + { + IntString _elem26; // required + _elem26 = new IntString(); + _elem26.read(iprot); + struct.lintString.add(_elem26); + } + iprot.readListEnd(); + } + struct.setLintStringIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 6: // M_STRING_STRING + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map27 = iprot.readMapBegin(); + struct.mStringString = new HashMap(2*_map27.size); + for (int _i28 = 0; _i28 < _map27.size; ++_i28) + { + String _key29; // required + String _val30; // required + _key29 = iprot.readString(); + _val30 = iprot.readString(); + struct.mStringString.put(_key29, _val30); + } + iprot.readMapEnd(); + } + struct.setMStringStringIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 7: // ATTRIBUTES + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map31 = iprot.readMapBegin(); + struct.attributes = new HashMap>>(2*_map31.size); + for (int _i32 = 0; _i32 < _map31.size; ++_i32) + { + String _key33; // required + Map> _val34; // required + _key33 = iprot.readString(); + { + org.apache.thrift.protocol.TMap _map35 = iprot.readMapBegin(); + _val34 = new HashMap>(2*_map35.size); + for (int _i36 = 0; _i36 < _map35.size; ++_i36) + { + String _key37; // required + Map _val38; // required + _key37 = iprot.readString(); + { + org.apache.thrift.protocol.TMap _map39 = iprot.readMapBegin(); + _val38 = new HashMap(2*_map39.size); + for (int _i40 = 0; _i40 < _map39.size; ++_i40) + { + String _key41; // required + PropValueUnion _val42; // required + _key41 = iprot.readString(); + _val42 = new PropValueUnion(); + _val42.read(iprot); + _val38.put(_key41, _val42); + } + iprot.readMapEnd(); + } + _val34.put(_key37, _val38); + } + iprot.readMapEnd(); + } + struct.attributes.put(_key33, _val34); + } + iprot.readMapEnd(); + } + struct.setAttributesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 8: // UNION_FIELD1 + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.unionField1 = new PropValueUnion(); + struct.unionField1.read(iprot); + struct.setUnionField1IsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 9: // UNION_FIELD2 + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.unionField2 = new PropValueUnion(); + struct.unionField2.read(iprot); + struct.setUnionField2IsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 10: // UNION_FIELD3 + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.unionField3 = new PropValueUnion(); + struct.unionField3.read(iprot); + struct.setUnionField3IsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, Complex struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(AINT_FIELD_DESC); + oprot.writeI32(struct.aint); + oprot.writeFieldEnd(); + if (struct.aString != null) { + oprot.writeFieldBegin(A_STRING_FIELD_DESC); + oprot.writeString(struct.aString); + oprot.writeFieldEnd(); + } + if (struct.lint != null) { + oprot.writeFieldBegin(LINT_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I32, struct.lint.size())); + for (int _iter43 : struct.lint) + { + oprot.writeI32(_iter43); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.lString != null) { + oprot.writeFieldBegin(L_STRING_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.lString.size())); + for (String _iter44 : struct.lString) + { + oprot.writeString(_iter44); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.lintString != null) { + oprot.writeFieldBegin(LINT_STRING_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.lintString.size())); + for (IntString _iter45 : struct.lintString) + { + _iter45.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.mStringString != null) { + oprot.writeFieldBegin(M_STRING_STRING_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.mStringString.size())); + for (Map.Entry _iter46 : struct.mStringString.entrySet()) + { + oprot.writeString(_iter46.getKey()); + oprot.writeString(_iter46.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.attributes != null) { + oprot.writeFieldBegin(ATTRIBUTES_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.MAP, struct.attributes.size())); + for (Map.Entry>> _iter47 : struct.attributes.entrySet()) + { + oprot.writeString(_iter47.getKey()); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.MAP, _iter47.getValue().size())); + for (Map.Entry> _iter48 : _iter47.getValue().entrySet()) + { + oprot.writeString(_iter48.getKey()); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, _iter48.getValue().size())); + for (Map.Entry _iter49 : _iter48.getValue().entrySet()) + { + oprot.writeString(_iter49.getKey()); + _iter49.getValue().write(oprot); + } + oprot.writeMapEnd(); + } + } + oprot.writeMapEnd(); + } + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.unionField1 != null) { + oprot.writeFieldBegin(UNION_FIELD1_FIELD_DESC); + struct.unionField1.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.unionField2 != null) { + oprot.writeFieldBegin(UNION_FIELD2_FIELD_DESC); + struct.unionField2.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.unionField3 != null) { + oprot.writeFieldBegin(UNION_FIELD3_FIELD_DESC); + struct.unionField3.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class ComplexTupleSchemeFactory implements SchemeFactory { + public ComplexTupleScheme getScheme() { + return new ComplexTupleScheme(); + } + } + + private static class ComplexTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, Complex struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetAint()) { + optionals.set(0); + } + if (struct.isSetAString()) { + optionals.set(1); + } + if (struct.isSetLint()) { + optionals.set(2); + } + if (struct.isSetLString()) { + optionals.set(3); + } + if (struct.isSetLintString()) { + optionals.set(4); + } + if (struct.isSetMStringString()) { + optionals.set(5); + } + if (struct.isSetAttributes()) { + optionals.set(6); + } + if (struct.isSetUnionField1()) { + optionals.set(7); + } + if (struct.isSetUnionField2()) { + optionals.set(8); + } + if (struct.isSetUnionField3()) { + optionals.set(9); + } + oprot.writeBitSet(optionals, 10); + if (struct.isSetAint()) { + oprot.writeI32(struct.aint); + } + if (struct.isSetAString()) { + oprot.writeString(struct.aString); + } + if (struct.isSetLint()) { + { + oprot.writeI32(struct.lint.size()); + for (int _iter50 : struct.lint) + { + oprot.writeI32(_iter50); + } + } + } + if (struct.isSetLString()) { + { + oprot.writeI32(struct.lString.size()); + for (String _iter51 : struct.lString) + { + oprot.writeString(_iter51); + } + } + } + if (struct.isSetLintString()) { + { + oprot.writeI32(struct.lintString.size()); + for (IntString _iter52 : struct.lintString) + { + _iter52.write(oprot); + } + } + } + if (struct.isSetMStringString()) { + { + oprot.writeI32(struct.mStringString.size()); + for (Map.Entry _iter53 : struct.mStringString.entrySet()) + { + oprot.writeString(_iter53.getKey()); + oprot.writeString(_iter53.getValue()); + } + } + } + if (struct.isSetAttributes()) { + { + oprot.writeI32(struct.attributes.size()); + for (Map.Entry>> _iter54 : struct.attributes.entrySet()) + { + oprot.writeString(_iter54.getKey()); + { + oprot.writeI32(_iter54.getValue().size()); + for (Map.Entry> _iter55 : _iter54.getValue().entrySet()) + { + oprot.writeString(_iter55.getKey()); + { + oprot.writeI32(_iter55.getValue().size()); + for (Map.Entry _iter56 : _iter55.getValue().entrySet()) + { + oprot.writeString(_iter56.getKey()); + _iter56.getValue().write(oprot); + } + } + } + } + } + } + } + if (struct.isSetUnionField1()) { + struct.unionField1.write(oprot); + } + if (struct.isSetUnionField2()) { + struct.unionField2.write(oprot); + } + if (struct.isSetUnionField3()) { + struct.unionField3.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, Complex struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(10); + if (incoming.get(0)) { + struct.aint = iprot.readI32(); + struct.setAintIsSet(true); + } + if (incoming.get(1)) { + struct.aString = iprot.readString(); + struct.setAStringIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TList _list57 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I32, iprot.readI32()); + struct.lint = new ArrayList(_list57.size); + for (int _i58 = 0; _i58 < _list57.size; ++_i58) + { + int _elem59; // required + _elem59 = iprot.readI32(); + struct.lint.add(_elem59); + } + } + struct.setLintIsSet(true); + } + if (incoming.get(3)) { + { + org.apache.thrift.protocol.TList _list60 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.lString = new ArrayList(_list60.size); + for (int _i61 = 0; _i61 < _list60.size; ++_i61) + { + String _elem62; // required + _elem62 = iprot.readString(); + struct.lString.add(_elem62); + } + } + struct.setLStringIsSet(true); + } + if (incoming.get(4)) { + { + org.apache.thrift.protocol.TList _list63 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.lintString = new ArrayList(_list63.size); + for (int _i64 = 0; _i64 < _list63.size; ++_i64) + { + IntString _elem65; // required + _elem65 = new IntString(); + _elem65.read(iprot); + struct.lintString.add(_elem65); + } + } + struct.setLintStringIsSet(true); + } + if (incoming.get(5)) { + { + org.apache.thrift.protocol.TMap _map66 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.mStringString = new HashMap(2*_map66.size); + for (int _i67 = 0; _i67 < _map66.size; ++_i67) + { + String _key68; // required + String _val69; // required + _key68 = iprot.readString(); + _val69 = iprot.readString(); + struct.mStringString.put(_key68, _val69); + } + } + struct.setMStringStringIsSet(true); + } + if (incoming.get(6)) { + { + org.apache.thrift.protocol.TMap _map70 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.MAP, iprot.readI32()); + struct.attributes = new HashMap>>(2*_map70.size); + for (int _i71 = 0; _i71 < _map70.size; ++_i71) + { + String _key72; // required + Map> _val73; // required + _key72 = iprot.readString(); + { + org.apache.thrift.protocol.TMap _map74 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.MAP, iprot.readI32()); + _val73 = new HashMap>(2*_map74.size); + for (int _i75 = 0; _i75 < _map74.size; ++_i75) + { + String _key76; // required + Map _val77; // required + _key76 = iprot.readString(); + { + org.apache.thrift.protocol.TMap _map78 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + _val77 = new HashMap(2*_map78.size); + for (int _i79 = 0; _i79 < _map78.size; ++_i79) + { + String _key80; // required + PropValueUnion _val81; // required + _key80 = iprot.readString(); + _val81 = new PropValueUnion(); + _val81.read(iprot); + _val77.put(_key80, _val81); + } + } + _val73.put(_key76, _val77); + } + } + struct.attributes.put(_key72, _val73); + } + } + struct.setAttributesIsSet(true); + } + if (incoming.get(7)) { + struct.unionField1 = new PropValueUnion(); + struct.unionField1.read(iprot); + struct.setUnionField1IsSet(true); + } + if (incoming.get(8)) { + struct.unionField2 = new PropValueUnion(); + struct.unionField2.read(iprot); + struct.setUnionField2IsSet(true); + } + if (incoming.get(9)) { + struct.unionField3 = new PropValueUnion(); + struct.unionField3.read(iprot); + struct.setUnionField3IsSet(true); + } + } + } + +} + diff --git a/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/IntString.java b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/IntString.java new file mode 100644 index 0000000000000000000000000000000000000000..bd580ad754f2545984dc53fff360b3a1742fbfae --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/IntString.java @@ -0,0 +1,591 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hive.serde2.thrift.test; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class IntString implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("IntString"); + + private static final org.apache.thrift.protocol.TField MYINT_FIELD_DESC = new org.apache.thrift.protocol.TField("myint", org.apache.thrift.protocol.TType.I32, (short)1); + private static final org.apache.thrift.protocol.TField MY_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("myString", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField UNDERSCORE_INT_FIELD_DESC = new org.apache.thrift.protocol.TField("underscore_int", org.apache.thrift.protocol.TType.I32, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new IntStringStandardSchemeFactory()); + schemes.put(TupleScheme.class, new IntStringTupleSchemeFactory()); + } + + private int myint; // required + private String myString; // required + private int underscore_int; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + MYINT((short)1, "myint"), + MY_STRING((short)2, "myString"), + UNDERSCORE_INT((short)3, "underscore_int"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // MYINT + return MYINT; + case 2: // MY_STRING + return MY_STRING; + case 3: // UNDERSCORE_INT + return UNDERSCORE_INT; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __MYINT_ISSET_ID = 0; + private static final int __UNDERSCORE_INT_ISSET_ID = 1; + private byte __isset_bitfield = 0; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.MYINT, new org.apache.thrift.meta_data.FieldMetaData("myint", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.MY_STRING, new org.apache.thrift.meta_data.FieldMetaData("myString", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.UNDERSCORE_INT, new org.apache.thrift.meta_data.FieldMetaData("underscore_int", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(IntString.class, metaDataMap); + } + + public IntString() { + } + + public IntString( + int myint, + String myString, + int underscore_int) + { + this(); + this.myint = myint; + setMyintIsSet(true); + this.myString = myString; + this.underscore_int = underscore_int; + setUnderscore_intIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public IntString(IntString other) { + __isset_bitfield = other.__isset_bitfield; + this.myint = other.myint; + if (other.isSetMyString()) { + this.myString = other.myString; + } + this.underscore_int = other.underscore_int; + } + + public IntString deepCopy() { + return new IntString(this); + } + + @Override + public void clear() { + setMyintIsSet(false); + this.myint = 0; + this.myString = null; + setUnderscore_intIsSet(false); + this.underscore_int = 0; + } + + public int getMyint() { + return this.myint; + } + + public void setMyint(int myint) { + this.myint = myint; + setMyintIsSet(true); + } + + public void unsetMyint() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MYINT_ISSET_ID); + } + + /** Returns true if field myint is set (has been assigned a value) and false otherwise */ + public boolean isSetMyint() { + return EncodingUtils.testBit(__isset_bitfield, __MYINT_ISSET_ID); + } + + public void setMyintIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MYINT_ISSET_ID, value); + } + + public String getMyString() { + return this.myString; + } + + public void setMyString(String myString) { + this.myString = myString; + } + + public void unsetMyString() { + this.myString = null; + } + + /** Returns true if field myString is set (has been assigned a value) and false otherwise */ + public boolean isSetMyString() { + return this.myString != null; + } + + public void setMyStringIsSet(boolean value) { + if (!value) { + this.myString = null; + } + } + + public int getUnderscore_int() { + return this.underscore_int; + } + + public void setUnderscore_int(int underscore_int) { + this.underscore_int = underscore_int; + setUnderscore_intIsSet(true); + } + + public void unsetUnderscore_int() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __UNDERSCORE_INT_ISSET_ID); + } + + /** Returns true if field underscore_int is set (has been assigned a value) and false otherwise */ + public boolean isSetUnderscore_int() { + return EncodingUtils.testBit(__isset_bitfield, __UNDERSCORE_INT_ISSET_ID); + } + + public void setUnderscore_intIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __UNDERSCORE_INT_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case MYINT: + if (value == null) { + unsetMyint(); + } else { + setMyint((Integer)value); + } + break; + + case MY_STRING: + if (value == null) { + unsetMyString(); + } else { + setMyString((String)value); + } + break; + + case UNDERSCORE_INT: + if (value == null) { + unsetUnderscore_int(); + } else { + setUnderscore_int((Integer)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case MYINT: + return Integer.valueOf(getMyint()); + + case MY_STRING: + return getMyString(); + + case UNDERSCORE_INT: + return Integer.valueOf(getUnderscore_int()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case MYINT: + return isSetMyint(); + case MY_STRING: + return isSetMyString(); + case UNDERSCORE_INT: + return isSetUnderscore_int(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof IntString) + return this.equals((IntString)that); + return false; + } + + public boolean equals(IntString that) { + if (that == null) + return false; + + boolean this_present_myint = true; + boolean that_present_myint = true; + if (this_present_myint || that_present_myint) { + if (!(this_present_myint && that_present_myint)) + return false; + if (this.myint != that.myint) + return false; + } + + boolean this_present_myString = true && this.isSetMyString(); + boolean that_present_myString = true && that.isSetMyString(); + if (this_present_myString || that_present_myString) { + if (!(this_present_myString && that_present_myString)) + return false; + if (!this.myString.equals(that.myString)) + return false; + } + + boolean this_present_underscore_int = true; + boolean that_present_underscore_int = true; + if (this_present_underscore_int || that_present_underscore_int) { + if (!(this_present_underscore_int && that_present_underscore_int)) + return false; + if (this.underscore_int != that.underscore_int) + return false; + } + + return true; + } + + @Override + public int hashCode() { + HashCodeBuilder builder = new HashCodeBuilder(); + + boolean present_myint = true; + builder.append(present_myint); + if (present_myint) + builder.append(myint); + + boolean present_myString = true && (isSetMyString()); + builder.append(present_myString); + if (present_myString) + builder.append(myString); + + boolean present_underscore_int = true; + builder.append(present_underscore_int); + if (present_underscore_int) + builder.append(underscore_int); + + return builder.toHashCode(); + } + + public int compareTo(IntString other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + IntString typedOther = (IntString)other; + + lastComparison = Boolean.valueOf(isSetMyint()).compareTo(typedOther.isSetMyint()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMyint()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.myint, typedOther.myint); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMyString()).compareTo(typedOther.isSetMyString()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMyString()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.myString, typedOther.myString); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetUnderscore_int()).compareTo(typedOther.isSetUnderscore_int()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetUnderscore_int()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.underscore_int, typedOther.underscore_int); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("IntString("); + boolean first = true; + + sb.append("myint:"); + sb.append(this.myint); + first = false; + if (!first) sb.append(", "); + sb.append("myString:"); + if (this.myString == null) { + sb.append("null"); + } else { + sb.append(this.myString); + } + first = false; + if (!first) sb.append(", "); + sb.append("underscore_int:"); + sb.append(this.underscore_int); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class IntStringStandardSchemeFactory implements SchemeFactory { + public IntStringStandardScheme getScheme() { + return new IntStringStandardScheme(); + } + } + + private static class IntStringStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, IntString struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // MYINT + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.myint = iprot.readI32(); + struct.setMyintIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // MY_STRING + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.myString = iprot.readString(); + struct.setMyStringIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // UNDERSCORE_INT + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.underscore_int = iprot.readI32(); + struct.setUnderscore_intIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, IntString struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(MYINT_FIELD_DESC); + oprot.writeI32(struct.myint); + oprot.writeFieldEnd(); + if (struct.myString != null) { + oprot.writeFieldBegin(MY_STRING_FIELD_DESC); + oprot.writeString(struct.myString); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(UNDERSCORE_INT_FIELD_DESC); + oprot.writeI32(struct.underscore_int); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class IntStringTupleSchemeFactory implements SchemeFactory { + public IntStringTupleScheme getScheme() { + return new IntStringTupleScheme(); + } + } + + private static class IntStringTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, IntString struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetMyint()) { + optionals.set(0); + } + if (struct.isSetMyString()) { + optionals.set(1); + } + if (struct.isSetUnderscore_int()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetMyint()) { + oprot.writeI32(struct.myint); + } + if (struct.isSetMyString()) { + oprot.writeString(struct.myString); + } + if (struct.isSetUnderscore_int()) { + oprot.writeI32(struct.underscore_int); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, IntString struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.myint = iprot.readI32(); + struct.setMyintIsSet(true); + } + if (incoming.get(1)) { + struct.myString = iprot.readString(); + struct.setMyStringIsSet(true); + } + if (incoming.get(2)) { + struct.underscore_int = iprot.readI32(); + struct.setUnderscore_intIsSet(true); + } + } + } + +} + diff --git a/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MegaStruct.java b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MegaStruct.java new file mode 100644 index 0000000000000000000000000000000000000000..386fef9568a50cbb70944bc51e2defd52ce6bc44 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MegaStruct.java @@ -0,0 +1,3145 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hive.serde2.thrift.test; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MegaStruct implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("MegaStruct"); + + private static final org.apache.thrift.protocol.TField MY_BOOL_FIELD_DESC = new org.apache.thrift.protocol.TField("my_bool", org.apache.thrift.protocol.TType.BOOL, (short)1); + private static final org.apache.thrift.protocol.TField MY_BYTE_FIELD_DESC = new org.apache.thrift.protocol.TField("my_byte", org.apache.thrift.protocol.TType.BYTE, (short)2); + private static final org.apache.thrift.protocol.TField MY_16BIT_INT_FIELD_DESC = new org.apache.thrift.protocol.TField("my_16bit_int", org.apache.thrift.protocol.TType.I16, (short)3); + private static final org.apache.thrift.protocol.TField MY_32BIT_INT_FIELD_DESC = new org.apache.thrift.protocol.TField("my_32bit_int", org.apache.thrift.protocol.TType.I32, (short)4); + private static final org.apache.thrift.protocol.TField MY_64BIT_INT_FIELD_DESC = new org.apache.thrift.protocol.TField("my_64bit_int", org.apache.thrift.protocol.TType.I64, (short)5); + private static final org.apache.thrift.protocol.TField MY_DOUBLE_FIELD_DESC = new org.apache.thrift.protocol.TField("my_double", org.apache.thrift.protocol.TType.DOUBLE, (short)6); + private static final org.apache.thrift.protocol.TField MY_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("my_string", org.apache.thrift.protocol.TType.STRING, (short)7); + private static final org.apache.thrift.protocol.TField MY_BINARY_FIELD_DESC = new org.apache.thrift.protocol.TField("my_binary", org.apache.thrift.protocol.TType.STRING, (short)8); + private static final org.apache.thrift.protocol.TField MY_STRING_STRING_MAP_FIELD_DESC = new org.apache.thrift.protocol.TField("my_string_string_map", org.apache.thrift.protocol.TType.MAP, (short)9); + private static final org.apache.thrift.protocol.TField MY_STRING_ENUM_MAP_FIELD_DESC = new org.apache.thrift.protocol.TField("my_string_enum_map", org.apache.thrift.protocol.TType.MAP, (short)10); + private static final org.apache.thrift.protocol.TField MY_ENUM_STRING_MAP_FIELD_DESC = new org.apache.thrift.protocol.TField("my_enum_string_map", org.apache.thrift.protocol.TType.MAP, (short)11); + private static final org.apache.thrift.protocol.TField MY_ENUM_STRUCT_MAP_FIELD_DESC = new org.apache.thrift.protocol.TField("my_enum_struct_map", org.apache.thrift.protocol.TType.MAP, (short)12); + private static final org.apache.thrift.protocol.TField MY_ENUM_STRINGLIST_MAP_FIELD_DESC = new org.apache.thrift.protocol.TField("my_enum_stringlist_map", org.apache.thrift.protocol.TType.MAP, (short)13); + private static final org.apache.thrift.protocol.TField MY_ENUM_STRUCTLIST_MAP_FIELD_DESC = new org.apache.thrift.protocol.TField("my_enum_structlist_map", org.apache.thrift.protocol.TType.MAP, (short)14); + private static final org.apache.thrift.protocol.TField MY_STRINGLIST_FIELD_DESC = new org.apache.thrift.protocol.TField("my_stringlist", org.apache.thrift.protocol.TType.LIST, (short)15); + private static final org.apache.thrift.protocol.TField MY_STRUCTLIST_FIELD_DESC = new org.apache.thrift.protocol.TField("my_structlist", org.apache.thrift.protocol.TType.LIST, (short)16); + private static final org.apache.thrift.protocol.TField MY_ENUMLIST_FIELD_DESC = new org.apache.thrift.protocol.TField("my_enumlist", org.apache.thrift.protocol.TType.LIST, (short)17); + private static final org.apache.thrift.protocol.TField MY_STRINGSET_FIELD_DESC = new org.apache.thrift.protocol.TField("my_stringset", org.apache.thrift.protocol.TType.SET, (short)18); + private static final org.apache.thrift.protocol.TField MY_ENUMSET_FIELD_DESC = new org.apache.thrift.protocol.TField("my_enumset", org.apache.thrift.protocol.TType.SET, (short)19); + private static final org.apache.thrift.protocol.TField MY_STRUCTSET_FIELD_DESC = new org.apache.thrift.protocol.TField("my_structset", org.apache.thrift.protocol.TType.SET, (short)20); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new MegaStructStandardSchemeFactory()); + schemes.put(TupleScheme.class, new MegaStructTupleSchemeFactory()); + } + + private boolean my_bool; // optional + private byte my_byte; // optional + private short my_16bit_int; // optional + private int my_32bit_int; // optional + private long my_64bit_int; // optional + private double my_double; // optional + private String my_string; // optional + private ByteBuffer my_binary; // optional + private Map my_string_string_map; // optional + private Map my_string_enum_map; // optional + private Map my_enum_string_map; // optional + private Map my_enum_struct_map; // optional + private Map> my_enum_stringlist_map; // optional + private Map> my_enum_structlist_map; // optional + private List my_stringlist; // optional + private List my_structlist; // optional + private List my_enumlist; // optional + private Set my_stringset; // optional + private Set my_enumset; // optional + private Set my_structset; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + MY_BOOL((short)1, "my_bool"), + MY_BYTE((short)2, "my_byte"), + MY_16BIT_INT((short)3, "my_16bit_int"), + MY_32BIT_INT((short)4, "my_32bit_int"), + MY_64BIT_INT((short)5, "my_64bit_int"), + MY_DOUBLE((short)6, "my_double"), + MY_STRING((short)7, "my_string"), + MY_BINARY((short)8, "my_binary"), + MY_STRING_STRING_MAP((short)9, "my_string_string_map"), + MY_STRING_ENUM_MAP((short)10, "my_string_enum_map"), + MY_ENUM_STRING_MAP((short)11, "my_enum_string_map"), + MY_ENUM_STRUCT_MAP((short)12, "my_enum_struct_map"), + MY_ENUM_STRINGLIST_MAP((short)13, "my_enum_stringlist_map"), + MY_ENUM_STRUCTLIST_MAP((short)14, "my_enum_structlist_map"), + MY_STRINGLIST((short)15, "my_stringlist"), + MY_STRUCTLIST((short)16, "my_structlist"), + MY_ENUMLIST((short)17, "my_enumlist"), + MY_STRINGSET((short)18, "my_stringset"), + MY_ENUMSET((short)19, "my_enumset"), + MY_STRUCTSET((short)20, "my_structset"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // MY_BOOL + return MY_BOOL; + case 2: // MY_BYTE + return MY_BYTE; + case 3: // MY_16BIT_INT + return MY_16BIT_INT; + case 4: // MY_32BIT_INT + return MY_32BIT_INT; + case 5: // MY_64BIT_INT + return MY_64BIT_INT; + case 6: // MY_DOUBLE + return MY_DOUBLE; + case 7: // MY_STRING + return MY_STRING; + case 8: // MY_BINARY + return MY_BINARY; + case 9: // MY_STRING_STRING_MAP + return MY_STRING_STRING_MAP; + case 10: // MY_STRING_ENUM_MAP + return MY_STRING_ENUM_MAP; + case 11: // MY_ENUM_STRING_MAP + return MY_ENUM_STRING_MAP; + case 12: // MY_ENUM_STRUCT_MAP + return MY_ENUM_STRUCT_MAP; + case 13: // MY_ENUM_STRINGLIST_MAP + return MY_ENUM_STRINGLIST_MAP; + case 14: // MY_ENUM_STRUCTLIST_MAP + return MY_ENUM_STRUCTLIST_MAP; + case 15: // MY_STRINGLIST + return MY_STRINGLIST; + case 16: // MY_STRUCTLIST + return MY_STRUCTLIST; + case 17: // MY_ENUMLIST + return MY_ENUMLIST; + case 18: // MY_STRINGSET + return MY_STRINGSET; + case 19: // MY_ENUMSET + return MY_ENUMSET; + case 20: // MY_STRUCTSET + return MY_STRUCTSET; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __MY_BOOL_ISSET_ID = 0; + private static final int __MY_BYTE_ISSET_ID = 1; + private static final int __MY_16BIT_INT_ISSET_ID = 2; + private static final int __MY_32BIT_INT_ISSET_ID = 3; + private static final int __MY_64BIT_INT_ISSET_ID = 4; + private static final int __MY_DOUBLE_ISSET_ID = 5; + private byte __isset_bitfield = 0; + private _Fields optionals[] = {_Fields.MY_BOOL,_Fields.MY_BYTE,_Fields.MY_16BIT_INT,_Fields.MY_32BIT_INT,_Fields.MY_64BIT_INT,_Fields.MY_DOUBLE,_Fields.MY_STRING,_Fields.MY_BINARY,_Fields.MY_STRING_STRING_MAP,_Fields.MY_STRING_ENUM_MAP,_Fields.MY_ENUM_STRING_MAP,_Fields.MY_ENUM_STRUCT_MAP,_Fields.MY_ENUM_STRINGLIST_MAP,_Fields.MY_ENUM_STRUCTLIST_MAP,_Fields.MY_STRINGLIST,_Fields.MY_STRUCTLIST,_Fields.MY_ENUMLIST,_Fields.MY_STRINGSET,_Fields.MY_ENUMSET,_Fields.MY_STRUCTSET}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.MY_BOOL, new org.apache.thrift.meta_data.FieldMetaData("my_bool", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + tmpMap.put(_Fields.MY_BYTE, new org.apache.thrift.meta_data.FieldMetaData("my_byte", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BYTE))); + tmpMap.put(_Fields.MY_16BIT_INT, new org.apache.thrift.meta_data.FieldMetaData("my_16bit_int", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I16))); + tmpMap.put(_Fields.MY_32BIT_INT, new org.apache.thrift.meta_data.FieldMetaData("my_32bit_int", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.MY_64BIT_INT, new org.apache.thrift.meta_data.FieldMetaData("my_64bit_int", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.MY_DOUBLE, new org.apache.thrift.meta_data.FieldMetaData("my_double", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE))); + tmpMap.put(_Fields.MY_STRING, new org.apache.thrift.meta_data.FieldMetaData("my_string", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.MY_BINARY, new org.apache.thrift.meta_data.FieldMetaData("my_binary", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); + tmpMap.put(_Fields.MY_STRING_STRING_MAP, new org.apache.thrift.meta_data.FieldMetaData("my_string_string_map", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + tmpMap.put(_Fields.MY_STRING_ENUM_MAP, new org.apache.thrift.meta_data.FieldMetaData("my_string_enum_map", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), + new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, MyEnum.class)))); + tmpMap.put(_Fields.MY_ENUM_STRING_MAP, new org.apache.thrift.meta_data.FieldMetaData("my_enum_string_map", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, MyEnum.class), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + tmpMap.put(_Fields.MY_ENUM_STRUCT_MAP, new org.apache.thrift.meta_data.FieldMetaData("my_enum_struct_map", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, MyEnum.class), + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, MiniStruct.class)))); + tmpMap.put(_Fields.MY_ENUM_STRINGLIST_MAP, new org.apache.thrift.meta_data.FieldMetaData("my_enum_stringlist_map", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, MyEnum.class), + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))))); + tmpMap.put(_Fields.MY_ENUM_STRUCTLIST_MAP, new org.apache.thrift.meta_data.FieldMetaData("my_enum_structlist_map", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, MyEnum.class), + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, MiniStruct.class))))); + tmpMap.put(_Fields.MY_STRINGLIST, new org.apache.thrift.meta_data.FieldMetaData("my_stringlist", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + tmpMap.put(_Fields.MY_STRUCTLIST, new org.apache.thrift.meta_data.FieldMetaData("my_structlist", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, MiniStruct.class)))); + tmpMap.put(_Fields.MY_ENUMLIST, new org.apache.thrift.meta_data.FieldMetaData("my_enumlist", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, MyEnum.class)))); + tmpMap.put(_Fields.MY_STRINGSET, new org.apache.thrift.meta_data.FieldMetaData("my_stringset", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.SetMetaData(org.apache.thrift.protocol.TType.SET, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + tmpMap.put(_Fields.MY_ENUMSET, new org.apache.thrift.meta_data.FieldMetaData("my_enumset", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.SetMetaData(org.apache.thrift.protocol.TType.SET, + new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, MyEnum.class)))); + tmpMap.put(_Fields.MY_STRUCTSET, new org.apache.thrift.meta_data.FieldMetaData("my_structset", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.SetMetaData(org.apache.thrift.protocol.TType.SET, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, MiniStruct.class)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(MegaStruct.class, metaDataMap); + } + + public MegaStruct() { + } + + /** + * Performs a deep copy on other. + */ + public MegaStruct(MegaStruct other) { + __isset_bitfield = other.__isset_bitfield; + this.my_bool = other.my_bool; + this.my_byte = other.my_byte; + this.my_16bit_int = other.my_16bit_int; + this.my_32bit_int = other.my_32bit_int; + this.my_64bit_int = other.my_64bit_int; + this.my_double = other.my_double; + if (other.isSetMy_string()) { + this.my_string = other.my_string; + } + if (other.isSetMy_binary()) { + this.my_binary = org.apache.thrift.TBaseHelper.copyBinary(other.my_binary); +; + } + if (other.isSetMy_string_string_map()) { + Map __this__my_string_string_map = new HashMap(); + for (Map.Entry other_element : other.my_string_string_map.entrySet()) { + + String other_element_key = other_element.getKey(); + String other_element_value = other_element.getValue(); + + String __this__my_string_string_map_copy_key = other_element_key; + + String __this__my_string_string_map_copy_value = other_element_value; + + __this__my_string_string_map.put(__this__my_string_string_map_copy_key, __this__my_string_string_map_copy_value); + } + this.my_string_string_map = __this__my_string_string_map; + } + if (other.isSetMy_string_enum_map()) { + Map __this__my_string_enum_map = new HashMap(); + for (Map.Entry other_element : other.my_string_enum_map.entrySet()) { + + String other_element_key = other_element.getKey(); + MyEnum other_element_value = other_element.getValue(); + + String __this__my_string_enum_map_copy_key = other_element_key; + + MyEnum __this__my_string_enum_map_copy_value = other_element_value; + + __this__my_string_enum_map.put(__this__my_string_enum_map_copy_key, __this__my_string_enum_map_copy_value); + } + this.my_string_enum_map = __this__my_string_enum_map; + } + if (other.isSetMy_enum_string_map()) { + Map __this__my_enum_string_map = new HashMap(); + for (Map.Entry other_element : other.my_enum_string_map.entrySet()) { + + MyEnum other_element_key = other_element.getKey(); + String other_element_value = other_element.getValue(); + + MyEnum __this__my_enum_string_map_copy_key = other_element_key; + + String __this__my_enum_string_map_copy_value = other_element_value; + + __this__my_enum_string_map.put(__this__my_enum_string_map_copy_key, __this__my_enum_string_map_copy_value); + } + this.my_enum_string_map = __this__my_enum_string_map; + } + if (other.isSetMy_enum_struct_map()) { + Map __this__my_enum_struct_map = new HashMap(); + for (Map.Entry other_element : other.my_enum_struct_map.entrySet()) { + + MyEnum other_element_key = other_element.getKey(); + MiniStruct other_element_value = other_element.getValue(); + + MyEnum __this__my_enum_struct_map_copy_key = other_element_key; + + MiniStruct __this__my_enum_struct_map_copy_value = new MiniStruct(other_element_value); + + __this__my_enum_struct_map.put(__this__my_enum_struct_map_copy_key, __this__my_enum_struct_map_copy_value); + } + this.my_enum_struct_map = __this__my_enum_struct_map; + } + if (other.isSetMy_enum_stringlist_map()) { + Map> __this__my_enum_stringlist_map = new HashMap>(); + for (Map.Entry> other_element : other.my_enum_stringlist_map.entrySet()) { + + MyEnum other_element_key = other_element.getKey(); + List other_element_value = other_element.getValue(); + + MyEnum __this__my_enum_stringlist_map_copy_key = other_element_key; + + List __this__my_enum_stringlist_map_copy_value = new ArrayList(); + for (String other_element_value_element : other_element_value) { + __this__my_enum_stringlist_map_copy_value.add(other_element_value_element); + } + + __this__my_enum_stringlist_map.put(__this__my_enum_stringlist_map_copy_key, __this__my_enum_stringlist_map_copy_value); + } + this.my_enum_stringlist_map = __this__my_enum_stringlist_map; + } + if (other.isSetMy_enum_structlist_map()) { + Map> __this__my_enum_structlist_map = new HashMap>(); + for (Map.Entry> other_element : other.my_enum_structlist_map.entrySet()) { + + MyEnum other_element_key = other_element.getKey(); + List other_element_value = other_element.getValue(); + + MyEnum __this__my_enum_structlist_map_copy_key = other_element_key; + + List __this__my_enum_structlist_map_copy_value = new ArrayList(); + for (MiniStruct other_element_value_element : other_element_value) { + __this__my_enum_structlist_map_copy_value.add(new MiniStruct(other_element_value_element)); + } + + __this__my_enum_structlist_map.put(__this__my_enum_structlist_map_copy_key, __this__my_enum_structlist_map_copy_value); + } + this.my_enum_structlist_map = __this__my_enum_structlist_map; + } + if (other.isSetMy_stringlist()) { + List __this__my_stringlist = new ArrayList(); + for (String other_element : other.my_stringlist) { + __this__my_stringlist.add(other_element); + } + this.my_stringlist = __this__my_stringlist; + } + if (other.isSetMy_structlist()) { + List __this__my_structlist = new ArrayList(); + for (MiniStruct other_element : other.my_structlist) { + __this__my_structlist.add(new MiniStruct(other_element)); + } + this.my_structlist = __this__my_structlist; + } + if (other.isSetMy_enumlist()) { + List __this__my_enumlist = new ArrayList(); + for (MyEnum other_element : other.my_enumlist) { + __this__my_enumlist.add(other_element); + } + this.my_enumlist = __this__my_enumlist; + } + if (other.isSetMy_stringset()) { + Set __this__my_stringset = new HashSet(); + for (String other_element : other.my_stringset) { + __this__my_stringset.add(other_element); + } + this.my_stringset = __this__my_stringset; + } + if (other.isSetMy_enumset()) { + Set __this__my_enumset = new HashSet(); + for (MyEnum other_element : other.my_enumset) { + __this__my_enumset.add(other_element); + } + this.my_enumset = __this__my_enumset; + } + if (other.isSetMy_structset()) { + Set __this__my_structset = new HashSet(); + for (MiniStruct other_element : other.my_structset) { + __this__my_structset.add(new MiniStruct(other_element)); + } + this.my_structset = __this__my_structset; + } + } + + public MegaStruct deepCopy() { + return new MegaStruct(this); + } + + @Override + public void clear() { + setMy_boolIsSet(false); + this.my_bool = false; + setMy_byteIsSet(false); + this.my_byte = 0; + setMy_16bit_intIsSet(false); + this.my_16bit_int = 0; + setMy_32bit_intIsSet(false); + this.my_32bit_int = 0; + setMy_64bit_intIsSet(false); + this.my_64bit_int = 0; + setMy_doubleIsSet(false); + this.my_double = 0.0; + this.my_string = null; + this.my_binary = null; + this.my_string_string_map = null; + this.my_string_enum_map = null; + this.my_enum_string_map = null; + this.my_enum_struct_map = null; + this.my_enum_stringlist_map = null; + this.my_enum_structlist_map = null; + this.my_stringlist = null; + this.my_structlist = null; + this.my_enumlist = null; + this.my_stringset = null; + this.my_enumset = null; + this.my_structset = null; + } + + public boolean isMy_bool() { + return this.my_bool; + } + + public void setMy_bool(boolean my_bool) { + this.my_bool = my_bool; + setMy_boolIsSet(true); + } + + public void unsetMy_bool() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MY_BOOL_ISSET_ID); + } + + /** Returns true if field my_bool is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_bool() { + return EncodingUtils.testBit(__isset_bitfield, __MY_BOOL_ISSET_ID); + } + + public void setMy_boolIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MY_BOOL_ISSET_ID, value); + } + + public byte getMy_byte() { + return this.my_byte; + } + + public void setMy_byte(byte my_byte) { + this.my_byte = my_byte; + setMy_byteIsSet(true); + } + + public void unsetMy_byte() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MY_BYTE_ISSET_ID); + } + + /** Returns true if field my_byte is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_byte() { + return EncodingUtils.testBit(__isset_bitfield, __MY_BYTE_ISSET_ID); + } + + public void setMy_byteIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MY_BYTE_ISSET_ID, value); + } + + public short getMy_16bit_int() { + return this.my_16bit_int; + } + + public void setMy_16bit_int(short my_16bit_int) { + this.my_16bit_int = my_16bit_int; + setMy_16bit_intIsSet(true); + } + + public void unsetMy_16bit_int() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MY_16BIT_INT_ISSET_ID); + } + + /** Returns true if field my_16bit_int is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_16bit_int() { + return EncodingUtils.testBit(__isset_bitfield, __MY_16BIT_INT_ISSET_ID); + } + + public void setMy_16bit_intIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MY_16BIT_INT_ISSET_ID, value); + } + + public int getMy_32bit_int() { + return this.my_32bit_int; + } + + public void setMy_32bit_int(int my_32bit_int) { + this.my_32bit_int = my_32bit_int; + setMy_32bit_intIsSet(true); + } + + public void unsetMy_32bit_int() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MY_32BIT_INT_ISSET_ID); + } + + /** Returns true if field my_32bit_int is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_32bit_int() { + return EncodingUtils.testBit(__isset_bitfield, __MY_32BIT_INT_ISSET_ID); + } + + public void setMy_32bit_intIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MY_32BIT_INT_ISSET_ID, value); + } + + public long getMy_64bit_int() { + return this.my_64bit_int; + } + + public void setMy_64bit_int(long my_64bit_int) { + this.my_64bit_int = my_64bit_int; + setMy_64bit_intIsSet(true); + } + + public void unsetMy_64bit_int() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MY_64BIT_INT_ISSET_ID); + } + + /** Returns true if field my_64bit_int is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_64bit_int() { + return EncodingUtils.testBit(__isset_bitfield, __MY_64BIT_INT_ISSET_ID); + } + + public void setMy_64bit_intIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MY_64BIT_INT_ISSET_ID, value); + } + + public double getMy_double() { + return this.my_double; + } + + public void setMy_double(double my_double) { + this.my_double = my_double; + setMy_doubleIsSet(true); + } + + public void unsetMy_double() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MY_DOUBLE_ISSET_ID); + } + + /** Returns true if field my_double is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_double() { + return EncodingUtils.testBit(__isset_bitfield, __MY_DOUBLE_ISSET_ID); + } + + public void setMy_doubleIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MY_DOUBLE_ISSET_ID, value); + } + + public String getMy_string() { + return this.my_string; + } + + public void setMy_string(String my_string) { + this.my_string = my_string; + } + + public void unsetMy_string() { + this.my_string = null; + } + + /** Returns true if field my_string is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_string() { + return this.my_string != null; + } + + public void setMy_stringIsSet(boolean value) { + if (!value) { + this.my_string = null; + } + } + + public byte[] getMy_binary() { + setMy_binary(org.apache.thrift.TBaseHelper.rightSize(my_binary)); + return my_binary == null ? null : my_binary.array(); + } + + public ByteBuffer bufferForMy_binary() { + return my_binary; + } + + public void setMy_binary(byte[] my_binary) { + setMy_binary(my_binary == null ? (ByteBuffer)null : ByteBuffer.wrap(my_binary)); + } + + public void setMy_binary(ByteBuffer my_binary) { + this.my_binary = my_binary; + } + + public void unsetMy_binary() { + this.my_binary = null; + } + + /** Returns true if field my_binary is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_binary() { + return this.my_binary != null; + } + + public void setMy_binaryIsSet(boolean value) { + if (!value) { + this.my_binary = null; + } + } + + public int getMy_string_string_mapSize() { + return (this.my_string_string_map == null) ? 0 : this.my_string_string_map.size(); + } + + public void putToMy_string_string_map(String key, String val) { + if (this.my_string_string_map == null) { + this.my_string_string_map = new HashMap(); + } + this.my_string_string_map.put(key, val); + } + + public Map getMy_string_string_map() { + return this.my_string_string_map; + } + + public void setMy_string_string_map(Map my_string_string_map) { + this.my_string_string_map = my_string_string_map; + } + + public void unsetMy_string_string_map() { + this.my_string_string_map = null; + } + + /** Returns true if field my_string_string_map is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_string_string_map() { + return this.my_string_string_map != null; + } + + public void setMy_string_string_mapIsSet(boolean value) { + if (!value) { + this.my_string_string_map = null; + } + } + + public int getMy_string_enum_mapSize() { + return (this.my_string_enum_map == null) ? 0 : this.my_string_enum_map.size(); + } + + public void putToMy_string_enum_map(String key, MyEnum val) { + if (this.my_string_enum_map == null) { + this.my_string_enum_map = new HashMap(); + } + this.my_string_enum_map.put(key, val); + } + + public Map getMy_string_enum_map() { + return this.my_string_enum_map; + } + + public void setMy_string_enum_map(Map my_string_enum_map) { + this.my_string_enum_map = my_string_enum_map; + } + + public void unsetMy_string_enum_map() { + this.my_string_enum_map = null; + } + + /** Returns true if field my_string_enum_map is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_string_enum_map() { + return this.my_string_enum_map != null; + } + + public void setMy_string_enum_mapIsSet(boolean value) { + if (!value) { + this.my_string_enum_map = null; + } + } + + public int getMy_enum_string_mapSize() { + return (this.my_enum_string_map == null) ? 0 : this.my_enum_string_map.size(); + } + + public void putToMy_enum_string_map(MyEnum key, String val) { + if (this.my_enum_string_map == null) { + this.my_enum_string_map = new HashMap(); + } + this.my_enum_string_map.put(key, val); + } + + public Map getMy_enum_string_map() { + return this.my_enum_string_map; + } + + public void setMy_enum_string_map(Map my_enum_string_map) { + this.my_enum_string_map = my_enum_string_map; + } + + public void unsetMy_enum_string_map() { + this.my_enum_string_map = null; + } + + /** Returns true if field my_enum_string_map is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_enum_string_map() { + return this.my_enum_string_map != null; + } + + public void setMy_enum_string_mapIsSet(boolean value) { + if (!value) { + this.my_enum_string_map = null; + } + } + + public int getMy_enum_struct_mapSize() { + return (this.my_enum_struct_map == null) ? 0 : this.my_enum_struct_map.size(); + } + + public void putToMy_enum_struct_map(MyEnum key, MiniStruct val) { + if (this.my_enum_struct_map == null) { + this.my_enum_struct_map = new HashMap(); + } + this.my_enum_struct_map.put(key, val); + } + + public Map getMy_enum_struct_map() { + return this.my_enum_struct_map; + } + + public void setMy_enum_struct_map(Map my_enum_struct_map) { + this.my_enum_struct_map = my_enum_struct_map; + } + + public void unsetMy_enum_struct_map() { + this.my_enum_struct_map = null; + } + + /** Returns true if field my_enum_struct_map is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_enum_struct_map() { + return this.my_enum_struct_map != null; + } + + public void setMy_enum_struct_mapIsSet(boolean value) { + if (!value) { + this.my_enum_struct_map = null; + } + } + + public int getMy_enum_stringlist_mapSize() { + return (this.my_enum_stringlist_map == null) ? 0 : this.my_enum_stringlist_map.size(); + } + + public void putToMy_enum_stringlist_map(MyEnum key, List val) { + if (this.my_enum_stringlist_map == null) { + this.my_enum_stringlist_map = new HashMap>(); + } + this.my_enum_stringlist_map.put(key, val); + } + + public Map> getMy_enum_stringlist_map() { + return this.my_enum_stringlist_map; + } + + public void setMy_enum_stringlist_map(Map> my_enum_stringlist_map) { + this.my_enum_stringlist_map = my_enum_stringlist_map; + } + + public void unsetMy_enum_stringlist_map() { + this.my_enum_stringlist_map = null; + } + + /** Returns true if field my_enum_stringlist_map is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_enum_stringlist_map() { + return this.my_enum_stringlist_map != null; + } + + public void setMy_enum_stringlist_mapIsSet(boolean value) { + if (!value) { + this.my_enum_stringlist_map = null; + } + } + + public int getMy_enum_structlist_mapSize() { + return (this.my_enum_structlist_map == null) ? 0 : this.my_enum_structlist_map.size(); + } + + public void putToMy_enum_structlist_map(MyEnum key, List val) { + if (this.my_enum_structlist_map == null) { + this.my_enum_structlist_map = new HashMap>(); + } + this.my_enum_structlist_map.put(key, val); + } + + public Map> getMy_enum_structlist_map() { + return this.my_enum_structlist_map; + } + + public void setMy_enum_structlist_map(Map> my_enum_structlist_map) { + this.my_enum_structlist_map = my_enum_structlist_map; + } + + public void unsetMy_enum_structlist_map() { + this.my_enum_structlist_map = null; + } + + /** Returns true if field my_enum_structlist_map is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_enum_structlist_map() { + return this.my_enum_structlist_map != null; + } + + public void setMy_enum_structlist_mapIsSet(boolean value) { + if (!value) { + this.my_enum_structlist_map = null; + } + } + + public int getMy_stringlistSize() { + return (this.my_stringlist == null) ? 0 : this.my_stringlist.size(); + } + + public java.util.Iterator getMy_stringlistIterator() { + return (this.my_stringlist == null) ? null : this.my_stringlist.iterator(); + } + + public void addToMy_stringlist(String elem) { + if (this.my_stringlist == null) { + this.my_stringlist = new ArrayList(); + } + this.my_stringlist.add(elem); + } + + public List getMy_stringlist() { + return this.my_stringlist; + } + + public void setMy_stringlist(List my_stringlist) { + this.my_stringlist = my_stringlist; + } + + public void unsetMy_stringlist() { + this.my_stringlist = null; + } + + /** Returns true if field my_stringlist is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_stringlist() { + return this.my_stringlist != null; + } + + public void setMy_stringlistIsSet(boolean value) { + if (!value) { + this.my_stringlist = null; + } + } + + public int getMy_structlistSize() { + return (this.my_structlist == null) ? 0 : this.my_structlist.size(); + } + + public java.util.Iterator getMy_structlistIterator() { + return (this.my_structlist == null) ? null : this.my_structlist.iterator(); + } + + public void addToMy_structlist(MiniStruct elem) { + if (this.my_structlist == null) { + this.my_structlist = new ArrayList(); + } + this.my_structlist.add(elem); + } + + public List getMy_structlist() { + return this.my_structlist; + } + + public void setMy_structlist(List my_structlist) { + this.my_structlist = my_structlist; + } + + public void unsetMy_structlist() { + this.my_structlist = null; + } + + /** Returns true if field my_structlist is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_structlist() { + return this.my_structlist != null; + } + + public void setMy_structlistIsSet(boolean value) { + if (!value) { + this.my_structlist = null; + } + } + + public int getMy_enumlistSize() { + return (this.my_enumlist == null) ? 0 : this.my_enumlist.size(); + } + + public java.util.Iterator getMy_enumlistIterator() { + return (this.my_enumlist == null) ? null : this.my_enumlist.iterator(); + } + + public void addToMy_enumlist(MyEnum elem) { + if (this.my_enumlist == null) { + this.my_enumlist = new ArrayList(); + } + this.my_enumlist.add(elem); + } + + public List getMy_enumlist() { + return this.my_enumlist; + } + + public void setMy_enumlist(List my_enumlist) { + this.my_enumlist = my_enumlist; + } + + public void unsetMy_enumlist() { + this.my_enumlist = null; + } + + /** Returns true if field my_enumlist is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_enumlist() { + return this.my_enumlist != null; + } + + public void setMy_enumlistIsSet(boolean value) { + if (!value) { + this.my_enumlist = null; + } + } + + public int getMy_stringsetSize() { + return (this.my_stringset == null) ? 0 : this.my_stringset.size(); + } + + public java.util.Iterator getMy_stringsetIterator() { + return (this.my_stringset == null) ? null : this.my_stringset.iterator(); + } + + public void addToMy_stringset(String elem) { + if (this.my_stringset == null) { + this.my_stringset = new HashSet(); + } + this.my_stringset.add(elem); + } + + public Set getMy_stringset() { + return this.my_stringset; + } + + public void setMy_stringset(Set my_stringset) { + this.my_stringset = my_stringset; + } + + public void unsetMy_stringset() { + this.my_stringset = null; + } + + /** Returns true if field my_stringset is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_stringset() { + return this.my_stringset != null; + } + + public void setMy_stringsetIsSet(boolean value) { + if (!value) { + this.my_stringset = null; + } + } + + public int getMy_enumsetSize() { + return (this.my_enumset == null) ? 0 : this.my_enumset.size(); + } + + public java.util.Iterator getMy_enumsetIterator() { + return (this.my_enumset == null) ? null : this.my_enumset.iterator(); + } + + public void addToMy_enumset(MyEnum elem) { + if (this.my_enumset == null) { + this.my_enumset = new HashSet(); + } + this.my_enumset.add(elem); + } + + public Set getMy_enumset() { + return this.my_enumset; + } + + public void setMy_enumset(Set my_enumset) { + this.my_enumset = my_enumset; + } + + public void unsetMy_enumset() { + this.my_enumset = null; + } + + /** Returns true if field my_enumset is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_enumset() { + return this.my_enumset != null; + } + + public void setMy_enumsetIsSet(boolean value) { + if (!value) { + this.my_enumset = null; + } + } + + public int getMy_structsetSize() { + return (this.my_structset == null) ? 0 : this.my_structset.size(); + } + + public java.util.Iterator getMy_structsetIterator() { + return (this.my_structset == null) ? null : this.my_structset.iterator(); + } + + public void addToMy_structset(MiniStruct elem) { + if (this.my_structset == null) { + this.my_structset = new HashSet(); + } + this.my_structset.add(elem); + } + + public Set getMy_structset() { + return this.my_structset; + } + + public void setMy_structset(Set my_structset) { + this.my_structset = my_structset; + } + + public void unsetMy_structset() { + this.my_structset = null; + } + + /** Returns true if field my_structset is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_structset() { + return this.my_structset != null; + } + + public void setMy_structsetIsSet(boolean value) { + if (!value) { + this.my_structset = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case MY_BOOL: + if (value == null) { + unsetMy_bool(); + } else { + setMy_bool((Boolean)value); + } + break; + + case MY_BYTE: + if (value == null) { + unsetMy_byte(); + } else { + setMy_byte((Byte)value); + } + break; + + case MY_16BIT_INT: + if (value == null) { + unsetMy_16bit_int(); + } else { + setMy_16bit_int((Short)value); + } + break; + + case MY_32BIT_INT: + if (value == null) { + unsetMy_32bit_int(); + } else { + setMy_32bit_int((Integer)value); + } + break; + + case MY_64BIT_INT: + if (value == null) { + unsetMy_64bit_int(); + } else { + setMy_64bit_int((Long)value); + } + break; + + case MY_DOUBLE: + if (value == null) { + unsetMy_double(); + } else { + setMy_double((Double)value); + } + break; + + case MY_STRING: + if (value == null) { + unsetMy_string(); + } else { + setMy_string((String)value); + } + break; + + case MY_BINARY: + if (value == null) { + unsetMy_binary(); + } else { + setMy_binary((ByteBuffer)value); + } + break; + + case MY_STRING_STRING_MAP: + if (value == null) { + unsetMy_string_string_map(); + } else { + setMy_string_string_map((Map)value); + } + break; + + case MY_STRING_ENUM_MAP: + if (value == null) { + unsetMy_string_enum_map(); + } else { + setMy_string_enum_map((Map)value); + } + break; + + case MY_ENUM_STRING_MAP: + if (value == null) { + unsetMy_enum_string_map(); + } else { + setMy_enum_string_map((Map)value); + } + break; + + case MY_ENUM_STRUCT_MAP: + if (value == null) { + unsetMy_enum_struct_map(); + } else { + setMy_enum_struct_map((Map)value); + } + break; + + case MY_ENUM_STRINGLIST_MAP: + if (value == null) { + unsetMy_enum_stringlist_map(); + } else { + setMy_enum_stringlist_map((Map>)value); + } + break; + + case MY_ENUM_STRUCTLIST_MAP: + if (value == null) { + unsetMy_enum_structlist_map(); + } else { + setMy_enum_structlist_map((Map>)value); + } + break; + + case MY_STRINGLIST: + if (value == null) { + unsetMy_stringlist(); + } else { + setMy_stringlist((List)value); + } + break; + + case MY_STRUCTLIST: + if (value == null) { + unsetMy_structlist(); + } else { + setMy_structlist((List)value); + } + break; + + case MY_ENUMLIST: + if (value == null) { + unsetMy_enumlist(); + } else { + setMy_enumlist((List)value); + } + break; + + case MY_STRINGSET: + if (value == null) { + unsetMy_stringset(); + } else { + setMy_stringset((Set)value); + } + break; + + case MY_ENUMSET: + if (value == null) { + unsetMy_enumset(); + } else { + setMy_enumset((Set)value); + } + break; + + case MY_STRUCTSET: + if (value == null) { + unsetMy_structset(); + } else { + setMy_structset((Set)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case MY_BOOL: + return Boolean.valueOf(isMy_bool()); + + case MY_BYTE: + return Byte.valueOf(getMy_byte()); + + case MY_16BIT_INT: + return Short.valueOf(getMy_16bit_int()); + + case MY_32BIT_INT: + return Integer.valueOf(getMy_32bit_int()); + + case MY_64BIT_INT: + return Long.valueOf(getMy_64bit_int()); + + case MY_DOUBLE: + return Double.valueOf(getMy_double()); + + case MY_STRING: + return getMy_string(); + + case MY_BINARY: + return getMy_binary(); + + case MY_STRING_STRING_MAP: + return getMy_string_string_map(); + + case MY_STRING_ENUM_MAP: + return getMy_string_enum_map(); + + case MY_ENUM_STRING_MAP: + return getMy_enum_string_map(); + + case MY_ENUM_STRUCT_MAP: + return getMy_enum_struct_map(); + + case MY_ENUM_STRINGLIST_MAP: + return getMy_enum_stringlist_map(); + + case MY_ENUM_STRUCTLIST_MAP: + return getMy_enum_structlist_map(); + + case MY_STRINGLIST: + return getMy_stringlist(); + + case MY_STRUCTLIST: + return getMy_structlist(); + + case MY_ENUMLIST: + return getMy_enumlist(); + + case MY_STRINGSET: + return getMy_stringset(); + + case MY_ENUMSET: + return getMy_enumset(); + + case MY_STRUCTSET: + return getMy_structset(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case MY_BOOL: + return isSetMy_bool(); + case MY_BYTE: + return isSetMy_byte(); + case MY_16BIT_INT: + return isSetMy_16bit_int(); + case MY_32BIT_INT: + return isSetMy_32bit_int(); + case MY_64BIT_INT: + return isSetMy_64bit_int(); + case MY_DOUBLE: + return isSetMy_double(); + case MY_STRING: + return isSetMy_string(); + case MY_BINARY: + return isSetMy_binary(); + case MY_STRING_STRING_MAP: + return isSetMy_string_string_map(); + case MY_STRING_ENUM_MAP: + return isSetMy_string_enum_map(); + case MY_ENUM_STRING_MAP: + return isSetMy_enum_string_map(); + case MY_ENUM_STRUCT_MAP: + return isSetMy_enum_struct_map(); + case MY_ENUM_STRINGLIST_MAP: + return isSetMy_enum_stringlist_map(); + case MY_ENUM_STRUCTLIST_MAP: + return isSetMy_enum_structlist_map(); + case MY_STRINGLIST: + return isSetMy_stringlist(); + case MY_STRUCTLIST: + return isSetMy_structlist(); + case MY_ENUMLIST: + return isSetMy_enumlist(); + case MY_STRINGSET: + return isSetMy_stringset(); + case MY_ENUMSET: + return isSetMy_enumset(); + case MY_STRUCTSET: + return isSetMy_structset(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof MegaStruct) + return this.equals((MegaStruct)that); + return false; + } + + public boolean equals(MegaStruct that) { + if (that == null) + return false; + + boolean this_present_my_bool = true && this.isSetMy_bool(); + boolean that_present_my_bool = true && that.isSetMy_bool(); + if (this_present_my_bool || that_present_my_bool) { + if (!(this_present_my_bool && that_present_my_bool)) + return false; + if (this.my_bool != that.my_bool) + return false; + } + + boolean this_present_my_byte = true && this.isSetMy_byte(); + boolean that_present_my_byte = true && that.isSetMy_byte(); + if (this_present_my_byte || that_present_my_byte) { + if (!(this_present_my_byte && that_present_my_byte)) + return false; + if (this.my_byte != that.my_byte) + return false; + } + + boolean this_present_my_16bit_int = true && this.isSetMy_16bit_int(); + boolean that_present_my_16bit_int = true && that.isSetMy_16bit_int(); + if (this_present_my_16bit_int || that_present_my_16bit_int) { + if (!(this_present_my_16bit_int && that_present_my_16bit_int)) + return false; + if (this.my_16bit_int != that.my_16bit_int) + return false; + } + + boolean this_present_my_32bit_int = true && this.isSetMy_32bit_int(); + boolean that_present_my_32bit_int = true && that.isSetMy_32bit_int(); + if (this_present_my_32bit_int || that_present_my_32bit_int) { + if (!(this_present_my_32bit_int && that_present_my_32bit_int)) + return false; + if (this.my_32bit_int != that.my_32bit_int) + return false; + } + + boolean this_present_my_64bit_int = true && this.isSetMy_64bit_int(); + boolean that_present_my_64bit_int = true && that.isSetMy_64bit_int(); + if (this_present_my_64bit_int || that_present_my_64bit_int) { + if (!(this_present_my_64bit_int && that_present_my_64bit_int)) + return false; + if (this.my_64bit_int != that.my_64bit_int) + return false; + } + + boolean this_present_my_double = true && this.isSetMy_double(); + boolean that_present_my_double = true && that.isSetMy_double(); + if (this_present_my_double || that_present_my_double) { + if (!(this_present_my_double && that_present_my_double)) + return false; + if (this.my_double != that.my_double) + return false; + } + + boolean this_present_my_string = true && this.isSetMy_string(); + boolean that_present_my_string = true && that.isSetMy_string(); + if (this_present_my_string || that_present_my_string) { + if (!(this_present_my_string && that_present_my_string)) + return false; + if (!this.my_string.equals(that.my_string)) + return false; + } + + boolean this_present_my_binary = true && this.isSetMy_binary(); + boolean that_present_my_binary = true && that.isSetMy_binary(); + if (this_present_my_binary || that_present_my_binary) { + if (!(this_present_my_binary && that_present_my_binary)) + return false; + if (!this.my_binary.equals(that.my_binary)) + return false; + } + + boolean this_present_my_string_string_map = true && this.isSetMy_string_string_map(); + boolean that_present_my_string_string_map = true && that.isSetMy_string_string_map(); + if (this_present_my_string_string_map || that_present_my_string_string_map) { + if (!(this_present_my_string_string_map && that_present_my_string_string_map)) + return false; + if (!this.my_string_string_map.equals(that.my_string_string_map)) + return false; + } + + boolean this_present_my_string_enum_map = true && this.isSetMy_string_enum_map(); + boolean that_present_my_string_enum_map = true && that.isSetMy_string_enum_map(); + if (this_present_my_string_enum_map || that_present_my_string_enum_map) { + if (!(this_present_my_string_enum_map && that_present_my_string_enum_map)) + return false; + if (!this.my_string_enum_map.equals(that.my_string_enum_map)) + return false; + } + + boolean this_present_my_enum_string_map = true && this.isSetMy_enum_string_map(); + boolean that_present_my_enum_string_map = true && that.isSetMy_enum_string_map(); + if (this_present_my_enum_string_map || that_present_my_enum_string_map) { + if (!(this_present_my_enum_string_map && that_present_my_enum_string_map)) + return false; + if (!this.my_enum_string_map.equals(that.my_enum_string_map)) + return false; + } + + boolean this_present_my_enum_struct_map = true && this.isSetMy_enum_struct_map(); + boolean that_present_my_enum_struct_map = true && that.isSetMy_enum_struct_map(); + if (this_present_my_enum_struct_map || that_present_my_enum_struct_map) { + if (!(this_present_my_enum_struct_map && that_present_my_enum_struct_map)) + return false; + if (!this.my_enum_struct_map.equals(that.my_enum_struct_map)) + return false; + } + + boolean this_present_my_enum_stringlist_map = true && this.isSetMy_enum_stringlist_map(); + boolean that_present_my_enum_stringlist_map = true && that.isSetMy_enum_stringlist_map(); + if (this_present_my_enum_stringlist_map || that_present_my_enum_stringlist_map) { + if (!(this_present_my_enum_stringlist_map && that_present_my_enum_stringlist_map)) + return false; + if (!this.my_enum_stringlist_map.equals(that.my_enum_stringlist_map)) + return false; + } + + boolean this_present_my_enum_structlist_map = true && this.isSetMy_enum_structlist_map(); + boolean that_present_my_enum_structlist_map = true && that.isSetMy_enum_structlist_map(); + if (this_present_my_enum_structlist_map || that_present_my_enum_structlist_map) { + if (!(this_present_my_enum_structlist_map && that_present_my_enum_structlist_map)) + return false; + if (!this.my_enum_structlist_map.equals(that.my_enum_structlist_map)) + return false; + } + + boolean this_present_my_stringlist = true && this.isSetMy_stringlist(); + boolean that_present_my_stringlist = true && that.isSetMy_stringlist(); + if (this_present_my_stringlist || that_present_my_stringlist) { + if (!(this_present_my_stringlist && that_present_my_stringlist)) + return false; + if (!this.my_stringlist.equals(that.my_stringlist)) + return false; + } + + boolean this_present_my_structlist = true && this.isSetMy_structlist(); + boolean that_present_my_structlist = true && that.isSetMy_structlist(); + if (this_present_my_structlist || that_present_my_structlist) { + if (!(this_present_my_structlist && that_present_my_structlist)) + return false; + if (!this.my_structlist.equals(that.my_structlist)) + return false; + } + + boolean this_present_my_enumlist = true && this.isSetMy_enumlist(); + boolean that_present_my_enumlist = true && that.isSetMy_enumlist(); + if (this_present_my_enumlist || that_present_my_enumlist) { + if (!(this_present_my_enumlist && that_present_my_enumlist)) + return false; + if (!this.my_enumlist.equals(that.my_enumlist)) + return false; + } + + boolean this_present_my_stringset = true && this.isSetMy_stringset(); + boolean that_present_my_stringset = true && that.isSetMy_stringset(); + if (this_present_my_stringset || that_present_my_stringset) { + if (!(this_present_my_stringset && that_present_my_stringset)) + return false; + if (!this.my_stringset.equals(that.my_stringset)) + return false; + } + + boolean this_present_my_enumset = true && this.isSetMy_enumset(); + boolean that_present_my_enumset = true && that.isSetMy_enumset(); + if (this_present_my_enumset || that_present_my_enumset) { + if (!(this_present_my_enumset && that_present_my_enumset)) + return false; + if (!this.my_enumset.equals(that.my_enumset)) + return false; + } + + boolean this_present_my_structset = true && this.isSetMy_structset(); + boolean that_present_my_structset = true && that.isSetMy_structset(); + if (this_present_my_structset || that_present_my_structset) { + if (!(this_present_my_structset && that_present_my_structset)) + return false; + if (!this.my_structset.equals(that.my_structset)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + HashCodeBuilder builder = new HashCodeBuilder(); + + boolean present_my_bool = true && (isSetMy_bool()); + builder.append(present_my_bool); + if (present_my_bool) + builder.append(my_bool); + + boolean present_my_byte = true && (isSetMy_byte()); + builder.append(present_my_byte); + if (present_my_byte) + builder.append(my_byte); + + boolean present_my_16bit_int = true && (isSetMy_16bit_int()); + builder.append(present_my_16bit_int); + if (present_my_16bit_int) + builder.append(my_16bit_int); + + boolean present_my_32bit_int = true && (isSetMy_32bit_int()); + builder.append(present_my_32bit_int); + if (present_my_32bit_int) + builder.append(my_32bit_int); + + boolean present_my_64bit_int = true && (isSetMy_64bit_int()); + builder.append(present_my_64bit_int); + if (present_my_64bit_int) + builder.append(my_64bit_int); + + boolean present_my_double = true && (isSetMy_double()); + builder.append(present_my_double); + if (present_my_double) + builder.append(my_double); + + boolean present_my_string = true && (isSetMy_string()); + builder.append(present_my_string); + if (present_my_string) + builder.append(my_string); + + boolean present_my_binary = true && (isSetMy_binary()); + builder.append(present_my_binary); + if (present_my_binary) + builder.append(my_binary); + + boolean present_my_string_string_map = true && (isSetMy_string_string_map()); + builder.append(present_my_string_string_map); + if (present_my_string_string_map) + builder.append(my_string_string_map); + + boolean present_my_string_enum_map = true && (isSetMy_string_enum_map()); + builder.append(present_my_string_enum_map); + if (present_my_string_enum_map) + builder.append(my_string_enum_map); + + boolean present_my_enum_string_map = true && (isSetMy_enum_string_map()); + builder.append(present_my_enum_string_map); + if (present_my_enum_string_map) + builder.append(my_enum_string_map); + + boolean present_my_enum_struct_map = true && (isSetMy_enum_struct_map()); + builder.append(present_my_enum_struct_map); + if (present_my_enum_struct_map) + builder.append(my_enum_struct_map); + + boolean present_my_enum_stringlist_map = true && (isSetMy_enum_stringlist_map()); + builder.append(present_my_enum_stringlist_map); + if (present_my_enum_stringlist_map) + builder.append(my_enum_stringlist_map); + + boolean present_my_enum_structlist_map = true && (isSetMy_enum_structlist_map()); + builder.append(present_my_enum_structlist_map); + if (present_my_enum_structlist_map) + builder.append(my_enum_structlist_map); + + boolean present_my_stringlist = true && (isSetMy_stringlist()); + builder.append(present_my_stringlist); + if (present_my_stringlist) + builder.append(my_stringlist); + + boolean present_my_structlist = true && (isSetMy_structlist()); + builder.append(present_my_structlist); + if (present_my_structlist) + builder.append(my_structlist); + + boolean present_my_enumlist = true && (isSetMy_enumlist()); + builder.append(present_my_enumlist); + if (present_my_enumlist) + builder.append(my_enumlist); + + boolean present_my_stringset = true && (isSetMy_stringset()); + builder.append(present_my_stringset); + if (present_my_stringset) + builder.append(my_stringset); + + boolean present_my_enumset = true && (isSetMy_enumset()); + builder.append(present_my_enumset); + if (present_my_enumset) + builder.append(my_enumset); + + boolean present_my_structset = true && (isSetMy_structset()); + builder.append(present_my_structset); + if (present_my_structset) + builder.append(my_structset); + + return builder.toHashCode(); + } + + public int compareTo(MegaStruct other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + MegaStruct typedOther = (MegaStruct)other; + + lastComparison = Boolean.valueOf(isSetMy_bool()).compareTo(typedOther.isSetMy_bool()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_bool()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_bool, typedOther.my_bool); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_byte()).compareTo(typedOther.isSetMy_byte()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_byte()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_byte, typedOther.my_byte); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_16bit_int()).compareTo(typedOther.isSetMy_16bit_int()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_16bit_int()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_16bit_int, typedOther.my_16bit_int); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_32bit_int()).compareTo(typedOther.isSetMy_32bit_int()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_32bit_int()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_32bit_int, typedOther.my_32bit_int); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_64bit_int()).compareTo(typedOther.isSetMy_64bit_int()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_64bit_int()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_64bit_int, typedOther.my_64bit_int); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_double()).compareTo(typedOther.isSetMy_double()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_double()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_double, typedOther.my_double); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_string()).compareTo(typedOther.isSetMy_string()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_string()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_string, typedOther.my_string); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_binary()).compareTo(typedOther.isSetMy_binary()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_binary()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_binary, typedOther.my_binary); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_string_string_map()).compareTo(typedOther.isSetMy_string_string_map()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_string_string_map()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_string_string_map, typedOther.my_string_string_map); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_string_enum_map()).compareTo(typedOther.isSetMy_string_enum_map()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_string_enum_map()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_string_enum_map, typedOther.my_string_enum_map); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_enum_string_map()).compareTo(typedOther.isSetMy_enum_string_map()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_enum_string_map()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_enum_string_map, typedOther.my_enum_string_map); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_enum_struct_map()).compareTo(typedOther.isSetMy_enum_struct_map()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_enum_struct_map()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_enum_struct_map, typedOther.my_enum_struct_map); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_enum_stringlist_map()).compareTo(typedOther.isSetMy_enum_stringlist_map()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_enum_stringlist_map()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_enum_stringlist_map, typedOther.my_enum_stringlist_map); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_enum_structlist_map()).compareTo(typedOther.isSetMy_enum_structlist_map()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_enum_structlist_map()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_enum_structlist_map, typedOther.my_enum_structlist_map); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_stringlist()).compareTo(typedOther.isSetMy_stringlist()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_stringlist()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_stringlist, typedOther.my_stringlist); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_structlist()).compareTo(typedOther.isSetMy_structlist()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_structlist()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_structlist, typedOther.my_structlist); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_enumlist()).compareTo(typedOther.isSetMy_enumlist()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_enumlist()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_enumlist, typedOther.my_enumlist); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_stringset()).compareTo(typedOther.isSetMy_stringset()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_stringset()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_stringset, typedOther.my_stringset); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_enumset()).compareTo(typedOther.isSetMy_enumset()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_enumset()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_enumset, typedOther.my_enumset); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_structset()).compareTo(typedOther.isSetMy_structset()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_structset()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_structset, typedOther.my_structset); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("MegaStruct("); + boolean first = true; + + if (isSetMy_bool()) { + sb.append("my_bool:"); + sb.append(this.my_bool); + first = false; + } + if (isSetMy_byte()) { + if (!first) sb.append(", "); + sb.append("my_byte:"); + sb.append(this.my_byte); + first = false; + } + if (isSetMy_16bit_int()) { + if (!first) sb.append(", "); + sb.append("my_16bit_int:"); + sb.append(this.my_16bit_int); + first = false; + } + if (isSetMy_32bit_int()) { + if (!first) sb.append(", "); + sb.append("my_32bit_int:"); + sb.append(this.my_32bit_int); + first = false; + } + if (isSetMy_64bit_int()) { + if (!first) sb.append(", "); + sb.append("my_64bit_int:"); + sb.append(this.my_64bit_int); + first = false; + } + if (isSetMy_double()) { + if (!first) sb.append(", "); + sb.append("my_double:"); + sb.append(this.my_double); + first = false; + } + if (isSetMy_string()) { + if (!first) sb.append(", "); + sb.append("my_string:"); + if (this.my_string == null) { + sb.append("null"); + } else { + sb.append(this.my_string); + } + first = false; + } + if (isSetMy_binary()) { + if (!first) sb.append(", "); + sb.append("my_binary:"); + if (this.my_binary == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.my_binary, sb); + } + first = false; + } + if (isSetMy_string_string_map()) { + if (!first) sb.append(", "); + sb.append("my_string_string_map:"); + if (this.my_string_string_map == null) { + sb.append("null"); + } else { + sb.append(this.my_string_string_map); + } + first = false; + } + if (isSetMy_string_enum_map()) { + if (!first) sb.append(", "); + sb.append("my_string_enum_map:"); + if (this.my_string_enum_map == null) { + sb.append("null"); + } else { + sb.append(this.my_string_enum_map); + } + first = false; + } + if (isSetMy_enum_string_map()) { + if (!first) sb.append(", "); + sb.append("my_enum_string_map:"); + if (this.my_enum_string_map == null) { + sb.append("null"); + } else { + sb.append(this.my_enum_string_map); + } + first = false; + } + if (isSetMy_enum_struct_map()) { + if (!first) sb.append(", "); + sb.append("my_enum_struct_map:"); + if (this.my_enum_struct_map == null) { + sb.append("null"); + } else { + sb.append(this.my_enum_struct_map); + } + first = false; + } + if (isSetMy_enum_stringlist_map()) { + if (!first) sb.append(", "); + sb.append("my_enum_stringlist_map:"); + if (this.my_enum_stringlist_map == null) { + sb.append("null"); + } else { + sb.append(this.my_enum_stringlist_map); + } + first = false; + } + if (isSetMy_enum_structlist_map()) { + if (!first) sb.append(", "); + sb.append("my_enum_structlist_map:"); + if (this.my_enum_structlist_map == null) { + sb.append("null"); + } else { + sb.append(this.my_enum_structlist_map); + } + first = false; + } + if (isSetMy_stringlist()) { + if (!first) sb.append(", "); + sb.append("my_stringlist:"); + if (this.my_stringlist == null) { + sb.append("null"); + } else { + sb.append(this.my_stringlist); + } + first = false; + } + if (isSetMy_structlist()) { + if (!first) sb.append(", "); + sb.append("my_structlist:"); + if (this.my_structlist == null) { + sb.append("null"); + } else { + sb.append(this.my_structlist); + } + first = false; + } + if (isSetMy_enumlist()) { + if (!first) sb.append(", "); + sb.append("my_enumlist:"); + if (this.my_enumlist == null) { + sb.append("null"); + } else { + sb.append(this.my_enumlist); + } + first = false; + } + if (isSetMy_stringset()) { + if (!first) sb.append(", "); + sb.append("my_stringset:"); + if (this.my_stringset == null) { + sb.append("null"); + } else { + sb.append(this.my_stringset); + } + first = false; + } + if (isSetMy_enumset()) { + if (!first) sb.append(", "); + sb.append("my_enumset:"); + if (this.my_enumset == null) { + sb.append("null"); + } else { + sb.append(this.my_enumset); + } + first = false; + } + if (isSetMy_structset()) { + if (!first) sb.append(", "); + sb.append("my_structset:"); + if (this.my_structset == null) { + sb.append("null"); + } else { + sb.append(this.my_structset); + } + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class MegaStructStandardSchemeFactory implements SchemeFactory { + public MegaStructStandardScheme getScheme() { + return new MegaStructStandardScheme(); + } + } + + private static class MegaStructStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, MegaStruct struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // MY_BOOL + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.my_bool = iprot.readBool(); + struct.setMy_boolIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // MY_BYTE + if (schemeField.type == org.apache.thrift.protocol.TType.BYTE) { + struct.my_byte = iprot.readByte(); + struct.setMy_byteIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // MY_16BIT_INT + if (schemeField.type == org.apache.thrift.protocol.TType.I16) { + struct.my_16bit_int = iprot.readI16(); + struct.setMy_16bit_intIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // MY_32BIT_INT + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.my_32bit_int = iprot.readI32(); + struct.setMy_32bit_intIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // MY_64BIT_INT + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.my_64bit_int = iprot.readI64(); + struct.setMy_64bit_intIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 6: // MY_DOUBLE + if (schemeField.type == org.apache.thrift.protocol.TType.DOUBLE) { + struct.my_double = iprot.readDouble(); + struct.setMy_doubleIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 7: // MY_STRING + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.my_string = iprot.readString(); + struct.setMy_stringIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 8: // MY_BINARY + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.my_binary = iprot.readBinary(); + struct.setMy_binaryIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 9: // MY_STRING_STRING_MAP + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map0 = iprot.readMapBegin(); + struct.my_string_string_map = new HashMap(2*_map0.size); + for (int _i1 = 0; _i1 < _map0.size; ++_i1) + { + String _key2; // required + String _val3; // required + _key2 = iprot.readString(); + _val3 = iprot.readString(); + struct.my_string_string_map.put(_key2, _val3); + } + iprot.readMapEnd(); + } + struct.setMy_string_string_mapIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 10: // MY_STRING_ENUM_MAP + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map4 = iprot.readMapBegin(); + struct.my_string_enum_map = new HashMap(2*_map4.size); + for (int _i5 = 0; _i5 < _map4.size; ++_i5) + { + String _key6; // required + MyEnum _val7; // required + _key6 = iprot.readString(); + _val7 = MyEnum.findByValue(iprot.readI32()); + struct.my_string_enum_map.put(_key6, _val7); + } + iprot.readMapEnd(); + } + struct.setMy_string_enum_mapIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 11: // MY_ENUM_STRING_MAP + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map8 = iprot.readMapBegin(); + struct.my_enum_string_map = new HashMap(2*_map8.size); + for (int _i9 = 0; _i9 < _map8.size; ++_i9) + { + MyEnum _key10; // required + String _val11; // required + _key10 = MyEnum.findByValue(iprot.readI32()); + _val11 = iprot.readString(); + struct.my_enum_string_map.put(_key10, _val11); + } + iprot.readMapEnd(); + } + struct.setMy_enum_string_mapIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 12: // MY_ENUM_STRUCT_MAP + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map12 = iprot.readMapBegin(); + struct.my_enum_struct_map = new HashMap(2*_map12.size); + for (int _i13 = 0; _i13 < _map12.size; ++_i13) + { + MyEnum _key14; // required + MiniStruct _val15; // required + _key14 = MyEnum.findByValue(iprot.readI32()); + _val15 = new MiniStruct(); + _val15.read(iprot); + struct.my_enum_struct_map.put(_key14, _val15); + } + iprot.readMapEnd(); + } + struct.setMy_enum_struct_mapIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 13: // MY_ENUM_STRINGLIST_MAP + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map16 = iprot.readMapBegin(); + struct.my_enum_stringlist_map = new HashMap>(2*_map16.size); + for (int _i17 = 0; _i17 < _map16.size; ++_i17) + { + MyEnum _key18; // required + List _val19; // required + _key18 = MyEnum.findByValue(iprot.readI32()); + { + org.apache.thrift.protocol.TList _list20 = iprot.readListBegin(); + _val19 = new ArrayList(_list20.size); + for (int _i21 = 0; _i21 < _list20.size; ++_i21) + { + String _elem22; // required + _elem22 = iprot.readString(); + _val19.add(_elem22); + } + iprot.readListEnd(); + } + struct.my_enum_stringlist_map.put(_key18, _val19); + } + iprot.readMapEnd(); + } + struct.setMy_enum_stringlist_mapIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 14: // MY_ENUM_STRUCTLIST_MAP + if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { + { + org.apache.thrift.protocol.TMap _map23 = iprot.readMapBegin(); + struct.my_enum_structlist_map = new HashMap>(2*_map23.size); + for (int _i24 = 0; _i24 < _map23.size; ++_i24) + { + MyEnum _key25; // required + List _val26; // required + _key25 = MyEnum.findByValue(iprot.readI32()); + { + org.apache.thrift.protocol.TList _list27 = iprot.readListBegin(); + _val26 = new ArrayList(_list27.size); + for (int _i28 = 0; _i28 < _list27.size; ++_i28) + { + MiniStruct _elem29; // required + _elem29 = new MiniStruct(); + _elem29.read(iprot); + _val26.add(_elem29); + } + iprot.readListEnd(); + } + struct.my_enum_structlist_map.put(_key25, _val26); + } + iprot.readMapEnd(); + } + struct.setMy_enum_structlist_mapIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 15: // MY_STRINGLIST + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list30 = iprot.readListBegin(); + struct.my_stringlist = new ArrayList(_list30.size); + for (int _i31 = 0; _i31 < _list30.size; ++_i31) + { + String _elem32; // required + _elem32 = iprot.readString(); + struct.my_stringlist.add(_elem32); + } + iprot.readListEnd(); + } + struct.setMy_stringlistIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 16: // MY_STRUCTLIST + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list33 = iprot.readListBegin(); + struct.my_structlist = new ArrayList(_list33.size); + for (int _i34 = 0; _i34 < _list33.size; ++_i34) + { + MiniStruct _elem35; // required + _elem35 = new MiniStruct(); + _elem35.read(iprot); + struct.my_structlist.add(_elem35); + } + iprot.readListEnd(); + } + struct.setMy_structlistIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 17: // MY_ENUMLIST + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list36 = iprot.readListBegin(); + struct.my_enumlist = new ArrayList(_list36.size); + for (int _i37 = 0; _i37 < _list36.size; ++_i37) + { + MyEnum _elem38; // required + _elem38 = MyEnum.findByValue(iprot.readI32()); + struct.my_enumlist.add(_elem38); + } + iprot.readListEnd(); + } + struct.setMy_enumlistIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 18: // MY_STRINGSET + if (schemeField.type == org.apache.thrift.protocol.TType.SET) { + { + org.apache.thrift.protocol.TSet _set39 = iprot.readSetBegin(); + struct.my_stringset = new HashSet(2*_set39.size); + for (int _i40 = 0; _i40 < _set39.size; ++_i40) + { + String _elem41; // required + _elem41 = iprot.readString(); + struct.my_stringset.add(_elem41); + } + iprot.readSetEnd(); + } + struct.setMy_stringsetIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 19: // MY_ENUMSET + if (schemeField.type == org.apache.thrift.protocol.TType.SET) { + { + org.apache.thrift.protocol.TSet _set42 = iprot.readSetBegin(); + struct.my_enumset = new HashSet(2*_set42.size); + for (int _i43 = 0; _i43 < _set42.size; ++_i43) + { + MyEnum _elem44; // required + _elem44 = MyEnum.findByValue(iprot.readI32()); + struct.my_enumset.add(_elem44); + } + iprot.readSetEnd(); + } + struct.setMy_enumsetIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 20: // MY_STRUCTSET + if (schemeField.type == org.apache.thrift.protocol.TType.SET) { + { + org.apache.thrift.protocol.TSet _set45 = iprot.readSetBegin(); + struct.my_structset = new HashSet(2*_set45.size); + for (int _i46 = 0; _i46 < _set45.size; ++_i46) + { + MiniStruct _elem47; // required + _elem47 = new MiniStruct(); + _elem47.read(iprot); + struct.my_structset.add(_elem47); + } + iprot.readSetEnd(); + } + struct.setMy_structsetIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, MegaStruct struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.isSetMy_bool()) { + oprot.writeFieldBegin(MY_BOOL_FIELD_DESC); + oprot.writeBool(struct.my_bool); + oprot.writeFieldEnd(); + } + if (struct.isSetMy_byte()) { + oprot.writeFieldBegin(MY_BYTE_FIELD_DESC); + oprot.writeByte(struct.my_byte); + oprot.writeFieldEnd(); + } + if (struct.isSetMy_16bit_int()) { + oprot.writeFieldBegin(MY_16BIT_INT_FIELD_DESC); + oprot.writeI16(struct.my_16bit_int); + oprot.writeFieldEnd(); + } + if (struct.isSetMy_32bit_int()) { + oprot.writeFieldBegin(MY_32BIT_INT_FIELD_DESC); + oprot.writeI32(struct.my_32bit_int); + oprot.writeFieldEnd(); + } + if (struct.isSetMy_64bit_int()) { + oprot.writeFieldBegin(MY_64BIT_INT_FIELD_DESC); + oprot.writeI64(struct.my_64bit_int); + oprot.writeFieldEnd(); + } + if (struct.isSetMy_double()) { + oprot.writeFieldBegin(MY_DOUBLE_FIELD_DESC); + oprot.writeDouble(struct.my_double); + oprot.writeFieldEnd(); + } + if (struct.my_string != null) { + if (struct.isSetMy_string()) { + oprot.writeFieldBegin(MY_STRING_FIELD_DESC); + oprot.writeString(struct.my_string); + oprot.writeFieldEnd(); + } + } + if (struct.my_binary != null) { + if (struct.isSetMy_binary()) { + oprot.writeFieldBegin(MY_BINARY_FIELD_DESC); + oprot.writeBinary(struct.my_binary); + oprot.writeFieldEnd(); + } + } + if (struct.my_string_string_map != null) { + if (struct.isSetMy_string_string_map()) { + oprot.writeFieldBegin(MY_STRING_STRING_MAP_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, struct.my_string_string_map.size())); + for (Map.Entry _iter48 : struct.my_string_string_map.entrySet()) + { + oprot.writeString(_iter48.getKey()); + oprot.writeString(_iter48.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + } + if (struct.my_string_enum_map != null) { + if (struct.isSetMy_string_enum_map()) { + oprot.writeFieldBegin(MY_STRING_ENUM_MAP_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, struct.my_string_enum_map.size())); + for (Map.Entry _iter49 : struct.my_string_enum_map.entrySet()) + { + oprot.writeString(_iter49.getKey()); + oprot.writeI32(_iter49.getValue().getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + } + if (struct.my_enum_string_map != null) { + if (struct.isSetMy_enum_string_map()) { + oprot.writeFieldBegin(MY_ENUM_STRING_MAP_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.I32, org.apache.thrift.protocol.TType.STRING, struct.my_enum_string_map.size())); + for (Map.Entry _iter50 : struct.my_enum_string_map.entrySet()) + { + oprot.writeI32(_iter50.getKey().getValue()); + oprot.writeString(_iter50.getValue()); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + } + if (struct.my_enum_struct_map != null) { + if (struct.isSetMy_enum_struct_map()) { + oprot.writeFieldBegin(MY_ENUM_STRUCT_MAP_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.I32, org.apache.thrift.protocol.TType.STRUCT, struct.my_enum_struct_map.size())); + for (Map.Entry _iter51 : struct.my_enum_struct_map.entrySet()) + { + oprot.writeI32(_iter51.getKey().getValue()); + _iter51.getValue().write(oprot); + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + } + if (struct.my_enum_stringlist_map != null) { + if (struct.isSetMy_enum_stringlist_map()) { + oprot.writeFieldBegin(MY_ENUM_STRINGLIST_MAP_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.I32, org.apache.thrift.protocol.TType.LIST, struct.my_enum_stringlist_map.size())); + for (Map.Entry> _iter52 : struct.my_enum_stringlist_map.entrySet()) + { + oprot.writeI32(_iter52.getKey().getValue()); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, _iter52.getValue().size())); + for (String _iter53 : _iter52.getValue()) + { + oprot.writeString(_iter53); + } + oprot.writeListEnd(); + } + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + } + if (struct.my_enum_structlist_map != null) { + if (struct.isSetMy_enum_structlist_map()) { + oprot.writeFieldBegin(MY_ENUM_STRUCTLIST_MAP_FIELD_DESC); + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.I32, org.apache.thrift.protocol.TType.LIST, struct.my_enum_structlist_map.size())); + for (Map.Entry> _iter54 : struct.my_enum_structlist_map.entrySet()) + { + oprot.writeI32(_iter54.getKey().getValue()); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, _iter54.getValue().size())); + for (MiniStruct _iter55 : _iter54.getValue()) + { + _iter55.write(oprot); + } + oprot.writeListEnd(); + } + } + oprot.writeMapEnd(); + } + oprot.writeFieldEnd(); + } + } + if (struct.my_stringlist != null) { + if (struct.isSetMy_stringlist()) { + oprot.writeFieldBegin(MY_STRINGLIST_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.my_stringlist.size())); + for (String _iter56 : struct.my_stringlist) + { + oprot.writeString(_iter56); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + } + if (struct.my_structlist != null) { + if (struct.isSetMy_structlist()) { + oprot.writeFieldBegin(MY_STRUCTLIST_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.my_structlist.size())); + for (MiniStruct _iter57 : struct.my_structlist) + { + _iter57.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + } + if (struct.my_enumlist != null) { + if (struct.isSetMy_enumlist()) { + oprot.writeFieldBegin(MY_ENUMLIST_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I32, struct.my_enumlist.size())); + for (MyEnum _iter58 : struct.my_enumlist) + { + oprot.writeI32(_iter58.getValue()); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + } + if (struct.my_stringset != null) { + if (struct.isSetMy_stringset()) { + oprot.writeFieldBegin(MY_STRINGSET_FIELD_DESC); + { + oprot.writeSetBegin(new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRING, struct.my_stringset.size())); + for (String _iter59 : struct.my_stringset) + { + oprot.writeString(_iter59); + } + oprot.writeSetEnd(); + } + oprot.writeFieldEnd(); + } + } + if (struct.my_enumset != null) { + if (struct.isSetMy_enumset()) { + oprot.writeFieldBegin(MY_ENUMSET_FIELD_DESC); + { + oprot.writeSetBegin(new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.I32, struct.my_enumset.size())); + for (MyEnum _iter60 : struct.my_enumset) + { + oprot.writeI32(_iter60.getValue()); + } + oprot.writeSetEnd(); + } + oprot.writeFieldEnd(); + } + } + if (struct.my_structset != null) { + if (struct.isSetMy_structset()) { + oprot.writeFieldBegin(MY_STRUCTSET_FIELD_DESC); + { + oprot.writeSetBegin(new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRUCT, struct.my_structset.size())); + for (MiniStruct _iter61 : struct.my_structset) + { + _iter61.write(oprot); + } + oprot.writeSetEnd(); + } + oprot.writeFieldEnd(); + } + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class MegaStructTupleSchemeFactory implements SchemeFactory { + public MegaStructTupleScheme getScheme() { + return new MegaStructTupleScheme(); + } + } + + private static class MegaStructTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, MegaStruct struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetMy_bool()) { + optionals.set(0); + } + if (struct.isSetMy_byte()) { + optionals.set(1); + } + if (struct.isSetMy_16bit_int()) { + optionals.set(2); + } + if (struct.isSetMy_32bit_int()) { + optionals.set(3); + } + if (struct.isSetMy_64bit_int()) { + optionals.set(4); + } + if (struct.isSetMy_double()) { + optionals.set(5); + } + if (struct.isSetMy_string()) { + optionals.set(6); + } + if (struct.isSetMy_binary()) { + optionals.set(7); + } + if (struct.isSetMy_string_string_map()) { + optionals.set(8); + } + if (struct.isSetMy_string_enum_map()) { + optionals.set(9); + } + if (struct.isSetMy_enum_string_map()) { + optionals.set(10); + } + if (struct.isSetMy_enum_struct_map()) { + optionals.set(11); + } + if (struct.isSetMy_enum_stringlist_map()) { + optionals.set(12); + } + if (struct.isSetMy_enum_structlist_map()) { + optionals.set(13); + } + if (struct.isSetMy_stringlist()) { + optionals.set(14); + } + if (struct.isSetMy_structlist()) { + optionals.set(15); + } + if (struct.isSetMy_enumlist()) { + optionals.set(16); + } + if (struct.isSetMy_stringset()) { + optionals.set(17); + } + if (struct.isSetMy_enumset()) { + optionals.set(18); + } + if (struct.isSetMy_structset()) { + optionals.set(19); + } + oprot.writeBitSet(optionals, 20); + if (struct.isSetMy_bool()) { + oprot.writeBool(struct.my_bool); + } + if (struct.isSetMy_byte()) { + oprot.writeByte(struct.my_byte); + } + if (struct.isSetMy_16bit_int()) { + oprot.writeI16(struct.my_16bit_int); + } + if (struct.isSetMy_32bit_int()) { + oprot.writeI32(struct.my_32bit_int); + } + if (struct.isSetMy_64bit_int()) { + oprot.writeI64(struct.my_64bit_int); + } + if (struct.isSetMy_double()) { + oprot.writeDouble(struct.my_double); + } + if (struct.isSetMy_string()) { + oprot.writeString(struct.my_string); + } + if (struct.isSetMy_binary()) { + oprot.writeBinary(struct.my_binary); + } + if (struct.isSetMy_string_string_map()) { + { + oprot.writeI32(struct.my_string_string_map.size()); + for (Map.Entry _iter62 : struct.my_string_string_map.entrySet()) + { + oprot.writeString(_iter62.getKey()); + oprot.writeString(_iter62.getValue()); + } + } + } + if (struct.isSetMy_string_enum_map()) { + { + oprot.writeI32(struct.my_string_enum_map.size()); + for (Map.Entry _iter63 : struct.my_string_enum_map.entrySet()) + { + oprot.writeString(_iter63.getKey()); + oprot.writeI32(_iter63.getValue().getValue()); + } + } + } + if (struct.isSetMy_enum_string_map()) { + { + oprot.writeI32(struct.my_enum_string_map.size()); + for (Map.Entry _iter64 : struct.my_enum_string_map.entrySet()) + { + oprot.writeI32(_iter64.getKey().getValue()); + oprot.writeString(_iter64.getValue()); + } + } + } + if (struct.isSetMy_enum_struct_map()) { + { + oprot.writeI32(struct.my_enum_struct_map.size()); + for (Map.Entry _iter65 : struct.my_enum_struct_map.entrySet()) + { + oprot.writeI32(_iter65.getKey().getValue()); + _iter65.getValue().write(oprot); + } + } + } + if (struct.isSetMy_enum_stringlist_map()) { + { + oprot.writeI32(struct.my_enum_stringlist_map.size()); + for (Map.Entry> _iter66 : struct.my_enum_stringlist_map.entrySet()) + { + oprot.writeI32(_iter66.getKey().getValue()); + { + oprot.writeI32(_iter66.getValue().size()); + for (String _iter67 : _iter66.getValue()) + { + oprot.writeString(_iter67); + } + } + } + } + } + if (struct.isSetMy_enum_structlist_map()) { + { + oprot.writeI32(struct.my_enum_structlist_map.size()); + for (Map.Entry> _iter68 : struct.my_enum_structlist_map.entrySet()) + { + oprot.writeI32(_iter68.getKey().getValue()); + { + oprot.writeI32(_iter68.getValue().size()); + for (MiniStruct _iter69 : _iter68.getValue()) + { + _iter69.write(oprot); + } + } + } + } + } + if (struct.isSetMy_stringlist()) { + { + oprot.writeI32(struct.my_stringlist.size()); + for (String _iter70 : struct.my_stringlist) + { + oprot.writeString(_iter70); + } + } + } + if (struct.isSetMy_structlist()) { + { + oprot.writeI32(struct.my_structlist.size()); + for (MiniStruct _iter71 : struct.my_structlist) + { + _iter71.write(oprot); + } + } + } + if (struct.isSetMy_enumlist()) { + { + oprot.writeI32(struct.my_enumlist.size()); + for (MyEnum _iter72 : struct.my_enumlist) + { + oprot.writeI32(_iter72.getValue()); + } + } + } + if (struct.isSetMy_stringset()) { + { + oprot.writeI32(struct.my_stringset.size()); + for (String _iter73 : struct.my_stringset) + { + oprot.writeString(_iter73); + } + } + } + if (struct.isSetMy_enumset()) { + { + oprot.writeI32(struct.my_enumset.size()); + for (MyEnum _iter74 : struct.my_enumset) + { + oprot.writeI32(_iter74.getValue()); + } + } + } + if (struct.isSetMy_structset()) { + { + oprot.writeI32(struct.my_structset.size()); + for (MiniStruct _iter75 : struct.my_structset) + { + _iter75.write(oprot); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, MegaStruct struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(20); + if (incoming.get(0)) { + struct.my_bool = iprot.readBool(); + struct.setMy_boolIsSet(true); + } + if (incoming.get(1)) { + struct.my_byte = iprot.readByte(); + struct.setMy_byteIsSet(true); + } + if (incoming.get(2)) { + struct.my_16bit_int = iprot.readI16(); + struct.setMy_16bit_intIsSet(true); + } + if (incoming.get(3)) { + struct.my_32bit_int = iprot.readI32(); + struct.setMy_32bit_intIsSet(true); + } + if (incoming.get(4)) { + struct.my_64bit_int = iprot.readI64(); + struct.setMy_64bit_intIsSet(true); + } + if (incoming.get(5)) { + struct.my_double = iprot.readDouble(); + struct.setMy_doubleIsSet(true); + } + if (incoming.get(6)) { + struct.my_string = iprot.readString(); + struct.setMy_stringIsSet(true); + } + if (incoming.get(7)) { + struct.my_binary = iprot.readBinary(); + struct.setMy_binaryIsSet(true); + } + if (incoming.get(8)) { + { + org.apache.thrift.protocol.TMap _map76 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.my_string_string_map = new HashMap(2*_map76.size); + for (int _i77 = 0; _i77 < _map76.size; ++_i77) + { + String _key78; // required + String _val79; // required + _key78 = iprot.readString(); + _val79 = iprot.readString(); + struct.my_string_string_map.put(_key78, _val79); + } + } + struct.setMy_string_string_mapIsSet(true); + } + if (incoming.get(9)) { + { + org.apache.thrift.protocol.TMap _map80 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.I32, iprot.readI32()); + struct.my_string_enum_map = new HashMap(2*_map80.size); + for (int _i81 = 0; _i81 < _map80.size; ++_i81) + { + String _key82; // required + MyEnum _val83; // required + _key82 = iprot.readString(); + _val83 = MyEnum.findByValue(iprot.readI32()); + struct.my_string_enum_map.put(_key82, _val83); + } + } + struct.setMy_string_enum_mapIsSet(true); + } + if (incoming.get(10)) { + { + org.apache.thrift.protocol.TMap _map84 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.I32, org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.my_enum_string_map = new HashMap(2*_map84.size); + for (int _i85 = 0; _i85 < _map84.size; ++_i85) + { + MyEnum _key86; // required + String _val87; // required + _key86 = MyEnum.findByValue(iprot.readI32()); + _val87 = iprot.readString(); + struct.my_enum_string_map.put(_key86, _val87); + } + } + struct.setMy_enum_string_mapIsSet(true); + } + if (incoming.get(11)) { + { + org.apache.thrift.protocol.TMap _map88 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.I32, org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.my_enum_struct_map = new HashMap(2*_map88.size); + for (int _i89 = 0; _i89 < _map88.size; ++_i89) + { + MyEnum _key90; // required + MiniStruct _val91; // required + _key90 = MyEnum.findByValue(iprot.readI32()); + _val91 = new MiniStruct(); + _val91.read(iprot); + struct.my_enum_struct_map.put(_key90, _val91); + } + } + struct.setMy_enum_struct_mapIsSet(true); + } + if (incoming.get(12)) { + { + org.apache.thrift.protocol.TMap _map92 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.I32, org.apache.thrift.protocol.TType.LIST, iprot.readI32()); + struct.my_enum_stringlist_map = new HashMap>(2*_map92.size); + for (int _i93 = 0; _i93 < _map92.size; ++_i93) + { + MyEnum _key94; // required + List _val95; // required + _key94 = MyEnum.findByValue(iprot.readI32()); + { + org.apache.thrift.protocol.TList _list96 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + _val95 = new ArrayList(_list96.size); + for (int _i97 = 0; _i97 < _list96.size; ++_i97) + { + String _elem98; // required + _elem98 = iprot.readString(); + _val95.add(_elem98); + } + } + struct.my_enum_stringlist_map.put(_key94, _val95); + } + } + struct.setMy_enum_stringlist_mapIsSet(true); + } + if (incoming.get(13)) { + { + org.apache.thrift.protocol.TMap _map99 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.I32, org.apache.thrift.protocol.TType.LIST, iprot.readI32()); + struct.my_enum_structlist_map = new HashMap>(2*_map99.size); + for (int _i100 = 0; _i100 < _map99.size; ++_i100) + { + MyEnum _key101; // required + List _val102; // required + _key101 = MyEnum.findByValue(iprot.readI32()); + { + org.apache.thrift.protocol.TList _list103 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + _val102 = new ArrayList(_list103.size); + for (int _i104 = 0; _i104 < _list103.size; ++_i104) + { + MiniStruct _elem105; // required + _elem105 = new MiniStruct(); + _elem105.read(iprot); + _val102.add(_elem105); + } + } + struct.my_enum_structlist_map.put(_key101, _val102); + } + } + struct.setMy_enum_structlist_mapIsSet(true); + } + if (incoming.get(14)) { + { + org.apache.thrift.protocol.TList _list106 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.my_stringlist = new ArrayList(_list106.size); + for (int _i107 = 0; _i107 < _list106.size; ++_i107) + { + String _elem108; // required + _elem108 = iprot.readString(); + struct.my_stringlist.add(_elem108); + } + } + struct.setMy_stringlistIsSet(true); + } + if (incoming.get(15)) { + { + org.apache.thrift.protocol.TList _list109 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.my_structlist = new ArrayList(_list109.size); + for (int _i110 = 0; _i110 < _list109.size; ++_i110) + { + MiniStruct _elem111; // required + _elem111 = new MiniStruct(); + _elem111.read(iprot); + struct.my_structlist.add(_elem111); + } + } + struct.setMy_structlistIsSet(true); + } + if (incoming.get(16)) { + { + org.apache.thrift.protocol.TList _list112 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I32, iprot.readI32()); + struct.my_enumlist = new ArrayList(_list112.size); + for (int _i113 = 0; _i113 < _list112.size; ++_i113) + { + MyEnum _elem114; // required + _elem114 = MyEnum.findByValue(iprot.readI32()); + struct.my_enumlist.add(_elem114); + } + } + struct.setMy_enumlistIsSet(true); + } + if (incoming.get(17)) { + { + org.apache.thrift.protocol.TSet _set115 = new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.my_stringset = new HashSet(2*_set115.size); + for (int _i116 = 0; _i116 < _set115.size; ++_i116) + { + String _elem117; // required + _elem117 = iprot.readString(); + struct.my_stringset.add(_elem117); + } + } + struct.setMy_stringsetIsSet(true); + } + if (incoming.get(18)) { + { + org.apache.thrift.protocol.TSet _set118 = new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.I32, iprot.readI32()); + struct.my_enumset = new HashSet(2*_set118.size); + for (int _i119 = 0; _i119 < _set118.size; ++_i119) + { + MyEnum _elem120; // required + _elem120 = MyEnum.findByValue(iprot.readI32()); + struct.my_enumset.add(_elem120); + } + } + struct.setMy_enumsetIsSet(true); + } + if (incoming.get(19)) { + { + org.apache.thrift.protocol.TSet _set121 = new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.my_structset = new HashSet(2*_set121.size); + for (int _i122 = 0; _i122 < _set121.size; ++_i122) + { + MiniStruct _elem123; // required + _elem123 = new MiniStruct(); + _elem123.read(iprot); + struct.my_structset.add(_elem123); + } + } + struct.setMy_structsetIsSet(true); + } + } + } + +} + diff --git a/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MiniStruct.java b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MiniStruct.java new file mode 100644 index 0000000000000000000000000000000000000000..b1d39465364802023a2cf5daf233f0869f25246a --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MiniStruct.java @@ -0,0 +1,507 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hive.serde2.thrift.test; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MiniStruct implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("MiniStruct"); + + private static final org.apache.thrift.protocol.TField MY_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("my_string", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField MY_ENUM_FIELD_DESC = new org.apache.thrift.protocol.TField("my_enum", org.apache.thrift.protocol.TType.I32, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new MiniStructStandardSchemeFactory()); + schemes.put(TupleScheme.class, new MiniStructTupleSchemeFactory()); + } + + private String my_string; // optional + private MyEnum my_enum; // optional + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + MY_STRING((short)1, "my_string"), + /** + * + * @see MyEnum + */ + MY_ENUM((short)2, "my_enum"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // MY_STRING + return MY_STRING; + case 2: // MY_ENUM + return MY_ENUM; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private _Fields optionals[] = {_Fields.MY_STRING,_Fields.MY_ENUM}; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.MY_STRING, new org.apache.thrift.meta_data.FieldMetaData("my_string", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.MY_ENUM, new org.apache.thrift.meta_data.FieldMetaData("my_enum", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, MyEnum.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(MiniStruct.class, metaDataMap); + } + + public MiniStruct() { + } + + /** + * Performs a deep copy on other. + */ + public MiniStruct(MiniStruct other) { + if (other.isSetMy_string()) { + this.my_string = other.my_string; + } + if (other.isSetMy_enum()) { + this.my_enum = other.my_enum; + } + } + + public MiniStruct deepCopy() { + return new MiniStruct(this); + } + + @Override + public void clear() { + this.my_string = null; + this.my_enum = null; + } + + public String getMy_string() { + return this.my_string; + } + + public void setMy_string(String my_string) { + this.my_string = my_string; + } + + public void unsetMy_string() { + this.my_string = null; + } + + /** Returns true if field my_string is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_string() { + return this.my_string != null; + } + + public void setMy_stringIsSet(boolean value) { + if (!value) { + this.my_string = null; + } + } + + /** + * + * @see MyEnum + */ + public MyEnum getMy_enum() { + return this.my_enum; + } + + /** + * + * @see MyEnum + */ + public void setMy_enum(MyEnum my_enum) { + this.my_enum = my_enum; + } + + public void unsetMy_enum() { + this.my_enum = null; + } + + /** Returns true if field my_enum is set (has been assigned a value) and false otherwise */ + public boolean isSetMy_enum() { + return this.my_enum != null; + } + + public void setMy_enumIsSet(boolean value) { + if (!value) { + this.my_enum = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case MY_STRING: + if (value == null) { + unsetMy_string(); + } else { + setMy_string((String)value); + } + break; + + case MY_ENUM: + if (value == null) { + unsetMy_enum(); + } else { + setMy_enum((MyEnum)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case MY_STRING: + return getMy_string(); + + case MY_ENUM: + return getMy_enum(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case MY_STRING: + return isSetMy_string(); + case MY_ENUM: + return isSetMy_enum(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof MiniStruct) + return this.equals((MiniStruct)that); + return false; + } + + public boolean equals(MiniStruct that) { + if (that == null) + return false; + + boolean this_present_my_string = true && this.isSetMy_string(); + boolean that_present_my_string = true && that.isSetMy_string(); + if (this_present_my_string || that_present_my_string) { + if (!(this_present_my_string && that_present_my_string)) + return false; + if (!this.my_string.equals(that.my_string)) + return false; + } + + boolean this_present_my_enum = true && this.isSetMy_enum(); + boolean that_present_my_enum = true && that.isSetMy_enum(); + if (this_present_my_enum || that_present_my_enum) { + if (!(this_present_my_enum && that_present_my_enum)) + return false; + if (!this.my_enum.equals(that.my_enum)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + HashCodeBuilder builder = new HashCodeBuilder(); + + boolean present_my_string = true && (isSetMy_string()); + builder.append(present_my_string); + if (present_my_string) + builder.append(my_string); + + boolean present_my_enum = true && (isSetMy_enum()); + builder.append(present_my_enum); + if (present_my_enum) + builder.append(my_enum.getValue()); + + return builder.toHashCode(); + } + + public int compareTo(MiniStruct other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + MiniStruct typedOther = (MiniStruct)other; + + lastComparison = Boolean.valueOf(isSetMy_string()).compareTo(typedOther.isSetMy_string()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_string()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_string, typedOther.my_string); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMy_enum()).compareTo(typedOther.isSetMy_enum()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMy_enum()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.my_enum, typedOther.my_enum); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("MiniStruct("); + boolean first = true; + + if (isSetMy_string()) { + sb.append("my_string:"); + if (this.my_string == null) { + sb.append("null"); + } else { + sb.append(this.my_string); + } + first = false; + } + if (isSetMy_enum()) { + if (!first) sb.append(", "); + sb.append("my_enum:"); + if (this.my_enum == null) { + sb.append("null"); + } else { + sb.append(this.my_enum); + } + first = false; + } + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class MiniStructStandardSchemeFactory implements SchemeFactory { + public MiniStructStandardScheme getScheme() { + return new MiniStructStandardScheme(); + } + } + + private static class MiniStructStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, MiniStruct struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // MY_STRING + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.my_string = iprot.readString(); + struct.setMy_stringIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // MY_ENUM + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.my_enum = MyEnum.findByValue(iprot.readI32()); + struct.setMy_enumIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, MiniStruct struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.my_string != null) { + if (struct.isSetMy_string()) { + oprot.writeFieldBegin(MY_STRING_FIELD_DESC); + oprot.writeString(struct.my_string); + oprot.writeFieldEnd(); + } + } + if (struct.my_enum != null) { + if (struct.isSetMy_enum()) { + oprot.writeFieldBegin(MY_ENUM_FIELD_DESC); + oprot.writeI32(struct.my_enum.getValue()); + oprot.writeFieldEnd(); + } + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class MiniStructTupleSchemeFactory implements SchemeFactory { + public MiniStructTupleScheme getScheme() { + return new MiniStructTupleScheme(); + } + } + + private static class MiniStructTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, MiniStruct struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetMy_string()) { + optionals.set(0); + } + if (struct.isSetMy_enum()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetMy_string()) { + oprot.writeString(struct.my_string); + } + if (struct.isSetMy_enum()) { + oprot.writeI32(struct.my_enum.getValue()); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, MiniStruct struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.my_string = iprot.readString(); + struct.setMy_stringIsSet(true); + } + if (incoming.get(1)) { + struct.my_enum = MyEnum.findByValue(iprot.readI32()); + struct.setMy_enumIsSet(true); + } + } + } + +} + diff --git a/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MyEnum.java b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MyEnum.java new file mode 100644 index 0000000000000000000000000000000000000000..5bda9ccb6ecc18c4550d0414d33f274cb548e8c2 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/MyEnum.java @@ -0,0 +1,45 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hive.serde2.thrift.test; + + +import java.util.Map; +import java.util.HashMap; +import org.apache.thrift.TEnum; + +public enum MyEnum implements org.apache.thrift.TEnum { + LLAMA(1), + ALPACA(2); + + private final int value; + + private MyEnum(int value) { + this.value = value; + } + + /** + * Get the integer value of this enum value, as defined in the Thrift IDL. + */ + public int getValue() { + return value; + } + + /** + * Find a the enum type by its integer value, as defined in the Thrift IDL. + * @return null if the value is not found. + */ + public static MyEnum findByValue(int value) { + switch (value) { + case 1: + return LLAMA; + case 2: + return ALPACA; + default: + return null; + } + } +} diff --git a/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/PropValueUnion.java b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/PropValueUnion.java new file mode 100644 index 0000000000000000000000000000000000000000..aa56dc91e083b05c92bb89fbe078e3b53d79516a --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/PropValueUnion.java @@ -0,0 +1,730 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hive.serde2.thrift.test; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PropValueUnion extends org.apache.thrift.TUnion { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("PropValueUnion"); + private static final org.apache.thrift.protocol.TField INT_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("intValue", org.apache.thrift.protocol.TType.I32, (short)1); + private static final org.apache.thrift.protocol.TField LONG_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("longValue", org.apache.thrift.protocol.TType.I64, (short)2); + private static final org.apache.thrift.protocol.TField STRING_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("stringValue", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField DOUBLE_VALUE_FIELD_DESC = new org.apache.thrift.protocol.TField("doubleValue", org.apache.thrift.protocol.TType.DOUBLE, (short)4); + private static final org.apache.thrift.protocol.TField FLAG_FIELD_DESC = new org.apache.thrift.protocol.TField("flag", org.apache.thrift.protocol.TType.BOOL, (short)5); + private static final org.apache.thrift.protocol.TField L_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("lString", org.apache.thrift.protocol.TType.LIST, (short)6); + private static final org.apache.thrift.protocol.TField UNION_MSTRING_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("unionMStringString", org.apache.thrift.protocol.TType.MAP, (short)7); + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + INT_VALUE((short)1, "intValue"), + LONG_VALUE((short)2, "longValue"), + STRING_VALUE((short)3, "stringValue"), + DOUBLE_VALUE((short)4, "doubleValue"), + FLAG((short)5, "flag"), + L_STRING((short)6, "lString"), + UNION_MSTRING_STRING((short)7, "unionMStringString"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // INT_VALUE + return INT_VALUE; + case 2: // LONG_VALUE + return LONG_VALUE; + case 3: // STRING_VALUE + return STRING_VALUE; + case 4: // DOUBLE_VALUE + return DOUBLE_VALUE; + case 5: // FLAG + return FLAG; + case 6: // L_STRING + return L_STRING; + case 7: // UNION_MSTRING_STRING + return UNION_MSTRING_STRING; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.INT_VALUE, new org.apache.thrift.meta_data.FieldMetaData("intValue", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.LONG_VALUE, new org.apache.thrift.meta_data.FieldMetaData("longValue", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.STRING_VALUE, new org.apache.thrift.meta_data.FieldMetaData("stringValue", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.DOUBLE_VALUE, new org.apache.thrift.meta_data.FieldMetaData("doubleValue", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE))); + tmpMap.put(_Fields.FLAG, new org.apache.thrift.meta_data.FieldMetaData("flag", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + tmpMap.put(_Fields.L_STRING, new org.apache.thrift.meta_data.FieldMetaData("lString", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + tmpMap.put(_Fields.UNION_MSTRING_STRING, new org.apache.thrift.meta_data.FieldMetaData("unionMStringString", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(PropValueUnion.class, metaDataMap); + } + + public PropValueUnion() { + super(); + } + + public PropValueUnion(_Fields setField, Object value) { + super(setField, value); + } + + public PropValueUnion(PropValueUnion other) { + super(other); + } + public PropValueUnion deepCopy() { + return new PropValueUnion(this); + } + + public static PropValueUnion intValue(int value) { + PropValueUnion x = new PropValueUnion(); + x.setIntValue(value); + return x; + } + + public static PropValueUnion longValue(long value) { + PropValueUnion x = new PropValueUnion(); + x.setLongValue(value); + return x; + } + + public static PropValueUnion stringValue(String value) { + PropValueUnion x = new PropValueUnion(); + x.setStringValue(value); + return x; + } + + public static PropValueUnion doubleValue(double value) { + PropValueUnion x = new PropValueUnion(); + x.setDoubleValue(value); + return x; + } + + public static PropValueUnion flag(boolean value) { + PropValueUnion x = new PropValueUnion(); + x.setFlag(value); + return x; + } + + public static PropValueUnion lString(List value) { + PropValueUnion x = new PropValueUnion(); + x.setLString(value); + return x; + } + + public static PropValueUnion unionMStringString(Map value) { + PropValueUnion x = new PropValueUnion(); + x.setUnionMStringString(value); + return x; + } + + + @Override + protected void checkType(_Fields setField, Object value) throws ClassCastException { + switch (setField) { + case INT_VALUE: + if (value instanceof Integer) { + break; + } + throw new ClassCastException("Was expecting value of type Integer for field 'intValue', but got " + value.getClass().getSimpleName()); + case LONG_VALUE: + if (value instanceof Long) { + break; + } + throw new ClassCastException("Was expecting value of type Long for field 'longValue', but got " + value.getClass().getSimpleName()); + case STRING_VALUE: + if (value instanceof String) { + break; + } + throw new ClassCastException("Was expecting value of type String for field 'stringValue', but got " + value.getClass().getSimpleName()); + case DOUBLE_VALUE: + if (value instanceof Double) { + break; + } + throw new ClassCastException("Was expecting value of type Double for field 'doubleValue', but got " + value.getClass().getSimpleName()); + case FLAG: + if (value instanceof Boolean) { + break; + } + throw new ClassCastException("Was expecting value of type Boolean for field 'flag', but got " + value.getClass().getSimpleName()); + case L_STRING: + if (value instanceof List) { + break; + } + throw new ClassCastException("Was expecting value of type List for field 'lString', but got " + value.getClass().getSimpleName()); + case UNION_MSTRING_STRING: + if (value instanceof Map) { + break; + } + throw new ClassCastException("Was expecting value of type Map for field 'unionMStringString', but got " + value.getClass().getSimpleName()); + default: + throw new IllegalArgumentException("Unknown field id " + setField); + } + } + + @Override + protected Object standardSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TField field) throws org.apache.thrift.TException { + _Fields setField = _Fields.findByThriftId(field.id); + if (setField != null) { + switch (setField) { + case INT_VALUE: + if (field.type == INT_VALUE_FIELD_DESC.type) { + Integer intValue; + intValue = iprot.readI32(); + return intValue; + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + return null; + } + case LONG_VALUE: + if (field.type == LONG_VALUE_FIELD_DESC.type) { + Long longValue; + longValue = iprot.readI64(); + return longValue; + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + return null; + } + case STRING_VALUE: + if (field.type == STRING_VALUE_FIELD_DESC.type) { + String stringValue; + stringValue = iprot.readString(); + return stringValue; + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + return null; + } + case DOUBLE_VALUE: + if (field.type == DOUBLE_VALUE_FIELD_DESC.type) { + Double doubleValue; + doubleValue = iprot.readDouble(); + return doubleValue; + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + return null; + } + case FLAG: + if (field.type == FLAG_FIELD_DESC.type) { + Boolean flag; + flag = iprot.readBool(); + return flag; + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + return null; + } + case L_STRING: + if (field.type == L_STRING_FIELD_DESC.type) { + List lString; + { + org.apache.thrift.protocol.TList _list0 = iprot.readListBegin(); + lString = new ArrayList(_list0.size); + for (int _i1 = 0; _i1 < _list0.size; ++_i1) + { + String _elem2; // required + _elem2 = iprot.readString(); + lString.add(_elem2); + } + iprot.readListEnd(); + } + return lString; + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + return null; + } + case UNION_MSTRING_STRING: + if (field.type == UNION_MSTRING_STRING_FIELD_DESC.type) { + Map unionMStringString; + { + org.apache.thrift.protocol.TMap _map3 = iprot.readMapBegin(); + unionMStringString = new HashMap(2*_map3.size); + for (int _i4 = 0; _i4 < _map3.size; ++_i4) + { + String _key5; // required + String _val6; // required + _key5 = iprot.readString(); + _val6 = iprot.readString(); + unionMStringString.put(_key5, _val6); + } + iprot.readMapEnd(); + } + return unionMStringString; + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + return null; + } + default: + throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!"); + } + } else { + return null; + } + } + + @Override + protected void standardSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + switch (setField_) { + case INT_VALUE: + Integer intValue = (Integer)value_; + oprot.writeI32(intValue); + return; + case LONG_VALUE: + Long longValue = (Long)value_; + oprot.writeI64(longValue); + return; + case STRING_VALUE: + String stringValue = (String)value_; + oprot.writeString(stringValue); + return; + case DOUBLE_VALUE: + Double doubleValue = (Double)value_; + oprot.writeDouble(doubleValue); + return; + case FLAG: + Boolean flag = (Boolean)value_; + oprot.writeBool(flag); + return; + case L_STRING: + List lString = (List)value_; + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, lString.size())); + for (String _iter7 : lString) + { + oprot.writeString(_iter7); + } + oprot.writeListEnd(); + } + return; + case UNION_MSTRING_STRING: + Map unionMStringString = (Map)value_; + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, unionMStringString.size())); + for (Map.Entry _iter8 : unionMStringString.entrySet()) + { + oprot.writeString(_iter8.getKey()); + oprot.writeString(_iter8.getValue()); + } + oprot.writeMapEnd(); + } + return; + default: + throw new IllegalStateException("Cannot write union with unknown field " + setField_); + } + } + + @Override + protected Object tupleSchemeReadValue(org.apache.thrift.protocol.TProtocol iprot, short fieldID) throws org.apache.thrift.TException { + _Fields setField = _Fields.findByThriftId(fieldID); + if (setField != null) { + switch (setField) { + case INT_VALUE: + Integer intValue; + intValue = iprot.readI32(); + return intValue; + case LONG_VALUE: + Long longValue; + longValue = iprot.readI64(); + return longValue; + case STRING_VALUE: + String stringValue; + stringValue = iprot.readString(); + return stringValue; + case DOUBLE_VALUE: + Double doubleValue; + doubleValue = iprot.readDouble(); + return doubleValue; + case FLAG: + Boolean flag; + flag = iprot.readBool(); + return flag; + case L_STRING: + List lString; + { + org.apache.thrift.protocol.TList _list9 = iprot.readListBegin(); + lString = new ArrayList(_list9.size); + for (int _i10 = 0; _i10 < _list9.size; ++_i10) + { + String _elem11; // required + _elem11 = iprot.readString(); + lString.add(_elem11); + } + iprot.readListEnd(); + } + return lString; + case UNION_MSTRING_STRING: + Map unionMStringString; + { + org.apache.thrift.protocol.TMap _map12 = iprot.readMapBegin(); + unionMStringString = new HashMap(2*_map12.size); + for (int _i13 = 0; _i13 < _map12.size; ++_i13) + { + String _key14; // required + String _val15; // required + _key14 = iprot.readString(); + _val15 = iprot.readString(); + unionMStringString.put(_key14, _val15); + } + iprot.readMapEnd(); + } + return unionMStringString; + default: + throw new IllegalStateException("setField wasn't null, but didn't match any of the case statements!"); + } + } else { + throw new TProtocolException("Couldn't find a field with field id " + fieldID); + } + } + + @Override + protected void tupleSchemeWriteValue(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + switch (setField_) { + case INT_VALUE: + Integer intValue = (Integer)value_; + oprot.writeI32(intValue); + return; + case LONG_VALUE: + Long longValue = (Long)value_; + oprot.writeI64(longValue); + return; + case STRING_VALUE: + String stringValue = (String)value_; + oprot.writeString(stringValue); + return; + case DOUBLE_VALUE: + Double doubleValue = (Double)value_; + oprot.writeDouble(doubleValue); + return; + case FLAG: + Boolean flag = (Boolean)value_; + oprot.writeBool(flag); + return; + case L_STRING: + List lString = (List)value_; + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, lString.size())); + for (String _iter16 : lString) + { + oprot.writeString(_iter16); + } + oprot.writeListEnd(); + } + return; + case UNION_MSTRING_STRING: + Map unionMStringString = (Map)value_; + { + oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, unionMStringString.size())); + for (Map.Entry _iter17 : unionMStringString.entrySet()) + { + oprot.writeString(_iter17.getKey()); + oprot.writeString(_iter17.getValue()); + } + oprot.writeMapEnd(); + } + return; + default: + throw new IllegalStateException("Cannot write union with unknown field " + setField_); + } + } + + @Override + protected org.apache.thrift.protocol.TField getFieldDesc(_Fields setField) { + switch (setField) { + case INT_VALUE: + return INT_VALUE_FIELD_DESC; + case LONG_VALUE: + return LONG_VALUE_FIELD_DESC; + case STRING_VALUE: + return STRING_VALUE_FIELD_DESC; + case DOUBLE_VALUE: + return DOUBLE_VALUE_FIELD_DESC; + case FLAG: + return FLAG_FIELD_DESC; + case L_STRING: + return L_STRING_FIELD_DESC; + case UNION_MSTRING_STRING: + return UNION_MSTRING_STRING_FIELD_DESC; + default: + throw new IllegalArgumentException("Unknown field id " + setField); + } + } + + @Override + protected org.apache.thrift.protocol.TStruct getStructDesc() { + return STRUCT_DESC; + } + + @Override + protected _Fields enumForId(short id) { + return _Fields.findByThriftIdOrThrow(id); + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + + public int getIntValue() { + if (getSetField() == _Fields.INT_VALUE) { + return (Integer)getFieldValue(); + } else { + throw new RuntimeException("Cannot get field 'intValue' because union is currently set to " + getFieldDesc(getSetField()).name); + } + } + + public void setIntValue(int value) { + setField_ = _Fields.INT_VALUE; + value_ = value; + } + + public long getLongValue() { + if (getSetField() == _Fields.LONG_VALUE) { + return (Long)getFieldValue(); + } else { + throw new RuntimeException("Cannot get field 'longValue' because union is currently set to " + getFieldDesc(getSetField()).name); + } + } + + public void setLongValue(long value) { + setField_ = _Fields.LONG_VALUE; + value_ = value; + } + + public String getStringValue() { + if (getSetField() == _Fields.STRING_VALUE) { + return (String)getFieldValue(); + } else { + throw new RuntimeException("Cannot get field 'stringValue' because union is currently set to " + getFieldDesc(getSetField()).name); + } + } + + public void setStringValue(String value) { + if (value == null) throw new NullPointerException(); + setField_ = _Fields.STRING_VALUE; + value_ = value; + } + + public double getDoubleValue() { + if (getSetField() == _Fields.DOUBLE_VALUE) { + return (Double)getFieldValue(); + } else { + throw new RuntimeException("Cannot get field 'doubleValue' because union is currently set to " + getFieldDesc(getSetField()).name); + } + } + + public void setDoubleValue(double value) { + setField_ = _Fields.DOUBLE_VALUE; + value_ = value; + } + + public boolean getFlag() { + if (getSetField() == _Fields.FLAG) { + return (Boolean)getFieldValue(); + } else { + throw new RuntimeException("Cannot get field 'flag' because union is currently set to " + getFieldDesc(getSetField()).name); + } + } + + public void setFlag(boolean value) { + setField_ = _Fields.FLAG; + value_ = value; + } + + public List getLString() { + if (getSetField() == _Fields.L_STRING) { + return (List)getFieldValue(); + } else { + throw new RuntimeException("Cannot get field 'lString' because union is currently set to " + getFieldDesc(getSetField()).name); + } + } + + public void setLString(List value) { + if (value == null) throw new NullPointerException(); + setField_ = _Fields.L_STRING; + value_ = value; + } + + public Map getUnionMStringString() { + if (getSetField() == _Fields.UNION_MSTRING_STRING) { + return (Map)getFieldValue(); + } else { + throw new RuntimeException("Cannot get field 'unionMStringString' because union is currently set to " + getFieldDesc(getSetField()).name); + } + } + + public void setUnionMStringString(Map value) { + if (value == null) throw new NullPointerException(); + setField_ = _Fields.UNION_MSTRING_STRING; + value_ = value; + } + + public boolean isSetIntValue() { + return setField_ == _Fields.INT_VALUE; + } + + + public boolean isSetLongValue() { + return setField_ == _Fields.LONG_VALUE; + } + + + public boolean isSetStringValue() { + return setField_ == _Fields.STRING_VALUE; + } + + + public boolean isSetDoubleValue() { + return setField_ == _Fields.DOUBLE_VALUE; + } + + + public boolean isSetFlag() { + return setField_ == _Fields.FLAG; + } + + + public boolean isSetLString() { + return setField_ == _Fields.L_STRING; + } + + + public boolean isSetUnionMStringString() { + return setField_ == _Fields.UNION_MSTRING_STRING; + } + + + public boolean equals(Object other) { + if (other instanceof PropValueUnion) { + return equals((PropValueUnion)other); + } else { + return false; + } + } + + public boolean equals(PropValueUnion other) { + return other != null && getSetField() == other.getSetField() && getFieldValue().equals(other.getFieldValue()); + } + + @Override + public int compareTo(PropValueUnion other) { + int lastComparison = org.apache.thrift.TBaseHelper.compareTo(getSetField(), other.getSetField()); + if (lastComparison == 0) { + return org.apache.thrift.TBaseHelper.compareTo(getFieldValue(), other.getFieldValue()); + } + return lastComparison; + } + + + @Override + public int hashCode() { + HashCodeBuilder hcb = new HashCodeBuilder(); + hcb.append(this.getClass().getName()); + org.apache.thrift.TFieldIdEnum setField = getSetField(); + if (setField != null) { + hcb.append(setField.getThriftFieldId()); + Object value = getFieldValue(); + if (value instanceof org.apache.thrift.TEnum) { + hcb.append(((org.apache.thrift.TEnum)getFieldValue()).getValue()); + } else { + hcb.append(value); + } + } + return hcb.toHashCode(); + } + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + +} diff --git a/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java new file mode 100644 index 0000000000000000000000000000000000000000..676f2b25427aee947ec71f73d71659e4cfab27e2 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java @@ -0,0 +1,549 @@ +/** + * Autogenerated by Thrift Compiler (0.9.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hive.serde2.thrift.test; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SetIntString implements org.apache.thrift.TBase, java.io.Serializable, Cloneable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("SetIntString"); + + private static final org.apache.thrift.protocol.TField S_INT_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("sIntString", org.apache.thrift.protocol.TType.SET, (short)1); + private static final org.apache.thrift.protocol.TField A_STRING_FIELD_DESC = new org.apache.thrift.protocol.TField("aString", org.apache.thrift.protocol.TType.STRING, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new SetIntStringStandardSchemeFactory()); + schemes.put(TupleScheme.class, new SetIntStringTupleSchemeFactory()); + } + + private Set sIntString; // required + private String aString; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + S_INT_STRING((short)1, "sIntString"), + A_STRING((short)2, "aString"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // S_INT_STRING + return S_INT_STRING; + case 2: // A_STRING + return A_STRING; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.S_INT_STRING, new org.apache.thrift.meta_data.FieldMetaData("sIntString", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.SetMetaData(org.apache.thrift.protocol.TType.SET, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, IntString.class)))); + tmpMap.put(_Fields.A_STRING, new org.apache.thrift.meta_data.FieldMetaData("aString", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(SetIntString.class, metaDataMap); + } + + public SetIntString() { + } + + public SetIntString( + Set sIntString, + String aString) + { + this(); + this.sIntString = sIntString; + this.aString = aString; + } + + /** + * Performs a deep copy on other. + */ + public SetIntString(SetIntString other) { + if (other.isSetSIntString()) { + Set __this__sIntString = new HashSet(); + for (IntString other_element : other.sIntString) { + __this__sIntString.add(new IntString(other_element)); + } + this.sIntString = __this__sIntString; + } + if (other.isSetAString()) { + this.aString = other.aString; + } + } + + public SetIntString deepCopy() { + return new SetIntString(this); + } + + @Override + public void clear() { + this.sIntString = null; + this.aString = null; + } + + public int getSIntStringSize() { + return (this.sIntString == null) ? 0 : this.sIntString.size(); + } + + public java.util.Iterator getSIntStringIterator() { + return (this.sIntString == null) ? null : this.sIntString.iterator(); + } + + public void addToSIntString(IntString elem) { + if (this.sIntString == null) { + this.sIntString = new HashSet(); + } + this.sIntString.add(elem); + } + + public Set getSIntString() { + return this.sIntString; + } + + public void setSIntString(Set sIntString) { + this.sIntString = sIntString; + } + + public void unsetSIntString() { + this.sIntString = null; + } + + /** Returns true if field sIntString is set (has been assigned a value) and false otherwise */ + public boolean isSetSIntString() { + return this.sIntString != null; + } + + public void setSIntStringIsSet(boolean value) { + if (!value) { + this.sIntString = null; + } + } + + public String getAString() { + return this.aString; + } + + public void setAString(String aString) { + this.aString = aString; + } + + public void unsetAString() { + this.aString = null; + } + + /** Returns true if field aString is set (has been assigned a value) and false otherwise */ + public boolean isSetAString() { + return this.aString != null; + } + + public void setAStringIsSet(boolean value) { + if (!value) { + this.aString = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case S_INT_STRING: + if (value == null) { + unsetSIntString(); + } else { + setSIntString((Set)value); + } + break; + + case A_STRING: + if (value == null) { + unsetAString(); + } else { + setAString((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case S_INT_STRING: + return getSIntString(); + + case A_STRING: + return getAString(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case S_INT_STRING: + return isSetSIntString(); + case A_STRING: + return isSetAString(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof SetIntString) + return this.equals((SetIntString)that); + return false; + } + + public boolean equals(SetIntString that) { + if (that == null) + return false; + + boolean this_present_sIntString = true && this.isSetSIntString(); + boolean that_present_sIntString = true && that.isSetSIntString(); + if (this_present_sIntString || that_present_sIntString) { + if (!(this_present_sIntString && that_present_sIntString)) + return false; + if (!this.sIntString.equals(that.sIntString)) + return false; + } + + boolean this_present_aString = true && this.isSetAString(); + boolean that_present_aString = true && that.isSetAString(); + if (this_present_aString || that_present_aString) { + if (!(this_present_aString && that_present_aString)) + return false; + if (!this.aString.equals(that.aString)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + HashCodeBuilder builder = new HashCodeBuilder(); + + boolean present_sIntString = true && (isSetSIntString()); + builder.append(present_sIntString); + if (present_sIntString) + builder.append(sIntString); + + boolean present_aString = true && (isSetAString()); + builder.append(present_aString); + if (present_aString) + builder.append(aString); + + return builder.toHashCode(); + } + + public int compareTo(SetIntString other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + SetIntString typedOther = (SetIntString)other; + + lastComparison = Boolean.valueOf(isSetSIntString()).compareTo(typedOther.isSetSIntString()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSIntString()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sIntString, typedOther.sIntString); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetAString()).compareTo(typedOther.isSetAString()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetAString()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.aString, typedOther.aString); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("SetIntString("); + boolean first = true; + + sb.append("sIntString:"); + if (this.sIntString == null) { + sb.append("null"); + } else { + sb.append(this.sIntString); + } + first = false; + if (!first) sb.append(", "); + sb.append("aString:"); + if (this.aString == null) { + sb.append("null"); + } else { + sb.append(this.aString); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class SetIntStringStandardSchemeFactory implements SchemeFactory { + public SetIntStringStandardScheme getScheme() { + return new SetIntStringStandardScheme(); + } + } + + private static class SetIntStringStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, SetIntString struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // S_INT_STRING + if (schemeField.type == org.apache.thrift.protocol.TType.SET) { + { + org.apache.thrift.protocol.TSet _set82 = iprot.readSetBegin(); + struct.sIntString = new HashSet(2*_set82.size); + for (int _i83 = 0; _i83 < _set82.size; ++_i83) + { + IntString _elem84; // required + _elem84 = new IntString(); + _elem84.read(iprot); + struct.sIntString.add(_elem84); + } + iprot.readSetEnd(); + } + struct.setSIntStringIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // A_STRING + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.aString = iprot.readString(); + struct.setAStringIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, SetIntString struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.sIntString != null) { + oprot.writeFieldBegin(S_INT_STRING_FIELD_DESC); + { + oprot.writeSetBegin(new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRUCT, struct.sIntString.size())); + for (IntString _iter85 : struct.sIntString) + { + _iter85.write(oprot); + } + oprot.writeSetEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.aString != null) { + oprot.writeFieldBegin(A_STRING_FIELD_DESC); + oprot.writeString(struct.aString); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class SetIntStringTupleSchemeFactory implements SchemeFactory { + public SetIntStringTupleScheme getScheme() { + return new SetIntStringTupleScheme(); + } + } + + private static class SetIntStringTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, SetIntString struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSIntString()) { + optionals.set(0); + } + if (struct.isSetAString()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSIntString()) { + { + oprot.writeI32(struct.sIntString.size()); + for (IntString _iter86 : struct.sIntString) + { + _iter86.write(oprot); + } + } + } + if (struct.isSetAString()) { + oprot.writeString(struct.aString); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, SetIntString struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TSet _set87 = new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); + struct.sIntString = new HashSet(2*_set87.size); + for (int _i88 = 0; _i88 < _set87.size; ++_i88) + { + IntString _elem89; // required + _elem89 = new IntString(); + _elem89.read(iprot); + struct.sIntString.add(_elem89); + } + } + struct.setSIntStringIsSet(true); + } + if (incoming.get(1)) { + struct.aString = iprot.readString(); + struct.setAStringIsSet(true); + } + } + } + +} + diff --git a/storage-api/serde-api/src/gen/thrift/gen-php/Types.php b/storage-api/serde-api/src/gen/thrift/gen-php/Types.php new file mode 100644 index 0000000000000000000000000000000000000000..8c2cea6b0da3fe33b1e07bfb6a097642b1bf03cd --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-php/Types.php @@ -0,0 +1,231 @@ + array( + 'var' => 'field0', + 'type' => TType::I32, + ), + ); + } + if (is_array($vals)) { + if (isset($vals['field0'])) { + $this->field0 = $vals['field0']; + } + } + } + + public function getName() { + return 'InnerStruct'; + } + + public function read($input) + { + $xfer = 0; + $fname = null; + $ftype = 0; + $fid = 0; + $xfer += $input->readStructBegin($fname); + while (true) + { + $xfer += $input->readFieldBegin($fname, $ftype, $fid); + if ($ftype == TType::STOP) { + break; + } + switch ($fid) + { + case 1: + if ($ftype == TType::I32) { + $xfer += $input->readI32($this->field0); + } else { + $xfer += $input->skip($ftype); + } + break; + default: + $xfer += $input->skip($ftype); + break; + } + $xfer += $input->readFieldEnd(); + } + $xfer += $input->readStructEnd(); + return $xfer; + } + + public function write($output) { + $xfer = 0; + $xfer += $output->writeStructBegin('InnerStruct'); + if ($this->field0 !== null) { + $xfer += $output->writeFieldBegin('field0', TType::I32, 1); + $xfer += $output->writeI32($this->field0); + $xfer += $output->writeFieldEnd(); + } + $xfer += $output->writeFieldStop(); + $xfer += $output->writeStructEnd(); + return $xfer; + } + +} + +class ThriftTestObj { + static $_TSPEC; + + public $field1 = null; + public $field2 = null; + public $field3 = null; + + public function __construct($vals=null) { + if (!isset(self::$_TSPEC)) { + self::$_TSPEC = array( + 1 => array( + 'var' => 'field1', + 'type' => TType::I32, + ), + 2 => array( + 'var' => 'field2', + 'type' => TType::STRING, + ), + 3 => array( + 'var' => 'field3', + 'type' => TType::LST, + 'etype' => TType::STRUCT, + 'elem' => array( + 'type' => TType::STRUCT, + 'class' => '\InnerStruct', + ), + ), + ); + } + if (is_array($vals)) { + if (isset($vals['field1'])) { + $this->field1 = $vals['field1']; + } + if (isset($vals['field2'])) { + $this->field2 = $vals['field2']; + } + if (isset($vals['field3'])) { + $this->field3 = $vals['field3']; + } + } + } + + public function getName() { + return 'ThriftTestObj'; + } + + public function read($input) + { + $xfer = 0; + $fname = null; + $ftype = 0; + $fid = 0; + $xfer += $input->readStructBegin($fname); + while (true) + { + $xfer += $input->readFieldBegin($fname, $ftype, $fid); + if ($ftype == TType::STOP) { + break; + } + switch ($fid) + { + case 1: + if ($ftype == TType::I32) { + $xfer += $input->readI32($this->field1); + } else { + $xfer += $input->skip($ftype); + } + break; + case 2: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->field2); + } else { + $xfer += $input->skip($ftype); + } + break; + case 3: + if ($ftype == TType::LST) { + $this->field3 = array(); + $_size0 = 0; + $_etype3 = 0; + $xfer += $input->readListBegin($_etype3, $_size0); + for ($_i4 = 0; $_i4 < $_size0; ++$_i4) + { + $elem5 = null; + $elem5 = new \InnerStruct(); + $xfer += $elem5->read($input); + $this->field3 []= $elem5; + } + $xfer += $input->readListEnd(); + } else { + $xfer += $input->skip($ftype); + } + break; + default: + $xfer += $input->skip($ftype); + break; + } + $xfer += $input->readFieldEnd(); + } + $xfer += $input->readStructEnd(); + return $xfer; + } + + public function write($output) { + $xfer = 0; + $xfer += $output->writeStructBegin('ThriftTestObj'); + if ($this->field1 !== null) { + $xfer += $output->writeFieldBegin('field1', TType::I32, 1); + $xfer += $output->writeI32($this->field1); + $xfer += $output->writeFieldEnd(); + } + if ($this->field2 !== null) { + $xfer += $output->writeFieldBegin('field2', TType::STRING, 2); + $xfer += $output->writeString($this->field2); + $xfer += $output->writeFieldEnd(); + } + if ($this->field3 !== null) { + if (!is_array($this->field3)) { + throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA); + } + $xfer += $output->writeFieldBegin('field3', TType::LST, 3); + { + $output->writeListBegin(TType::STRUCT, count($this->field3)); + { + foreach ($this->field3 as $iter6) + { + $xfer += $iter6->write($output); + } + } + $output->writeListEnd(); + } + $xfer += $output->writeFieldEnd(); + } + $xfer += $output->writeFieldStop(); + $xfer += $output->writeStructEnd(); + return $xfer; + } + +} + + diff --git a/storage-api/serde-api/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php b/storage-api/serde-api/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php new file mode 100644 index 0000000000000000000000000000000000000000..8b8bb00fdddeccee4b02e9aacccaf7ccab492ae6 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-php/org/apache/hadoop/hive/serde/Types.php @@ -0,0 +1,145 @@ + true, + "boolean" => true, + "tinyint" => true, + "smallint" => true, + "int" => true, + "bigint" => true, + "float" => true, + "double" => true, + "string" => true, + "varchar" => true, + "char" => true, + "date" => true, + "datetime" => true, + "timestamp" => true, + "interval_year_month" => true, + "interval_day_time" => true, + "decimal" => true, + "binary" => true, + "timestamp with local time zone" => true, +); + +$GLOBALS['serde_CONSTANTS']['CollectionTypes'] = array( + "array" => true, + "map" => true, +); + +$GLOBALS['serde_CONSTANTS']['IntegralTypes'] = array( + "tinyint" => true, + "smallint" => true, + "int" => true, + "bigint" => true, +); + + diff --git a/storage-api/serde-api/src/gen/thrift/gen-py/__init__.py b/storage-api/serde-api/src/gen/thrift/gen-py/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/storage-api/serde-api/src/gen/thrift/gen-py/complex/__init__.py b/storage-api/serde-api/src/gen/thrift/gen-py/complex/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..adefd8e51fcf17862843d83fd7ea34546ef09fca --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-py/complex/__init__.py @@ -0,0 +1 @@ +__all__ = ['ttypes', 'constants'] diff --git a/storage-api/serde-api/src/gen/thrift/gen-py/complex/constants.py b/storage-api/serde-api/src/gen/thrift/gen-py/complex/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..a236326cf38cf20a4d300bc6eb2eff0be93ef2e7 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-py/complex/constants.py @@ -0,0 +1,11 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TException, TApplicationException +from ttypes import * + diff --git a/storage-api/serde-api/src/gen/thrift/gen-py/complex/ttypes.py b/storage-api/serde-api/src/gen/thrift/gen-py/complex/ttypes.py new file mode 100644 index 0000000000000000000000000000000000000000..c2748d0fd9c41743f7d4d30aed74026c7715ba88 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-py/complex/ttypes.py @@ -0,0 +1,570 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TException, TApplicationException + +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol, TProtocol +try: + from thrift.protocol import fastbinary +except: + fastbinary = None + + + +class PropValueUnion: + """ + Attributes: + - intValue + - longValue + - stringValue + - doubleValue + - flag + - lString + - unionMStringString + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'intValue', None, None, ), # 1 + (2, TType.I64, 'longValue', None, None, ), # 2 + (3, TType.STRING, 'stringValue', None, None, ), # 3 + (4, TType.DOUBLE, 'doubleValue', None, None, ), # 4 + (5, TType.BOOL, 'flag', None, None, ), # 5 + (6, TType.LIST, 'lString', (TType.STRING,None), None, ), # 6 + (7, TType.MAP, 'unionMStringString', (TType.STRING,None,TType.STRING,None), None, ), # 7 + ) + + def __init__(self, intValue=None, longValue=None, stringValue=None, doubleValue=None, flag=None, lString=None, unionMStringString=None,): + self.intValue = intValue + self.longValue = longValue + self.stringValue = stringValue + self.doubleValue = doubleValue + self.flag = flag + self.lString = lString + self.unionMStringString = unionMStringString + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.intValue = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.longValue = iprot.readI64(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.stringValue = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.DOUBLE: + self.doubleValue = iprot.readDouble(); + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.BOOL: + self.flag = iprot.readBool(); + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.LIST: + self.lString = [] + (_etype3, _size0) = iprot.readListBegin() + for _i4 in xrange(_size0): + _elem5 = iprot.readString(); + self.lString.append(_elem5) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.MAP: + self.unionMStringString = {} + (_ktype7, _vtype8, _size6 ) = iprot.readMapBegin() + for _i10 in xrange(_size6): + _key11 = iprot.readString(); + _val12 = iprot.readString(); + self.unionMStringString[_key11] = _val12 + iprot.readMapEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('PropValueUnion') + if self.intValue is not None: + oprot.writeFieldBegin('intValue', TType.I32, 1) + oprot.writeI32(self.intValue) + oprot.writeFieldEnd() + if self.longValue is not None: + oprot.writeFieldBegin('longValue', TType.I64, 2) + oprot.writeI64(self.longValue) + oprot.writeFieldEnd() + if self.stringValue is not None: + oprot.writeFieldBegin('stringValue', TType.STRING, 3) + oprot.writeString(self.stringValue) + oprot.writeFieldEnd() + if self.doubleValue is not None: + oprot.writeFieldBegin('doubleValue', TType.DOUBLE, 4) + oprot.writeDouble(self.doubleValue) + oprot.writeFieldEnd() + if self.flag is not None: + oprot.writeFieldBegin('flag', TType.BOOL, 5) + oprot.writeBool(self.flag) + oprot.writeFieldEnd() + if self.lString is not None: + oprot.writeFieldBegin('lString', TType.LIST, 6) + oprot.writeListBegin(TType.STRING, len(self.lString)) + for iter13 in self.lString: + oprot.writeString(iter13) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.unionMStringString is not None: + oprot.writeFieldBegin('unionMStringString', TType.MAP, 7) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.unionMStringString)) + for kiter14,viter15 in self.unionMStringString.items(): + oprot.writeString(kiter14) + oprot.writeString(viter15) + oprot.writeMapEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class IntString: + """ + Attributes: + - myint + - myString + - underscore_int + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'myint', None, None, ), # 1 + (2, TType.STRING, 'myString', None, None, ), # 2 + (3, TType.I32, 'underscore_int', None, None, ), # 3 + ) + + def __init__(self, myint=None, myString=None, underscore_int=None,): + self.myint = myint + self.myString = myString + self.underscore_int = underscore_int + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.myint = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.myString = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.underscore_int = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('IntString') + if self.myint is not None: + oprot.writeFieldBegin('myint', TType.I32, 1) + oprot.writeI32(self.myint) + oprot.writeFieldEnd() + if self.myString is not None: + oprot.writeFieldBegin('myString', TType.STRING, 2) + oprot.writeString(self.myString) + oprot.writeFieldEnd() + if self.underscore_int is not None: + oprot.writeFieldBegin('underscore_int', TType.I32, 3) + oprot.writeI32(self.underscore_int) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class Complex: + """ + Attributes: + - aint + - aString + - lint + - lString + - lintString + - mStringString + - attributes + - unionField1 + - unionField2 + - unionField3 + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'aint', None, None, ), # 1 + (2, TType.STRING, 'aString', None, None, ), # 2 + (3, TType.LIST, 'lint', (TType.I32,None), None, ), # 3 + (4, TType.LIST, 'lString', (TType.STRING,None), None, ), # 4 + (5, TType.LIST, 'lintString', (TType.STRUCT,(IntString, IntString.thrift_spec)), None, ), # 5 + (6, TType.MAP, 'mStringString', (TType.STRING,None,TType.STRING,None), None, ), # 6 + (7, TType.MAP, 'attributes', (TType.STRING,None,TType.MAP,(TType.STRING,None,TType.MAP,(TType.STRING,None,TType.STRUCT,(PropValueUnion, PropValueUnion.thrift_spec)))), None, ), # 7 + (8, TType.STRUCT, 'unionField1', (PropValueUnion, PropValueUnion.thrift_spec), None, ), # 8 + (9, TType.STRUCT, 'unionField2', (PropValueUnion, PropValueUnion.thrift_spec), None, ), # 9 + (10, TType.STRUCT, 'unionField3', (PropValueUnion, PropValueUnion.thrift_spec), None, ), # 10 + ) + + def __init__(self, aint=None, aString=None, lint=None, lString=None, lintString=None, mStringString=None, attributes=None, unionField1=None, unionField2=None, unionField3=None,): + self.aint = aint + self.aString = aString + self.lint = lint + self.lString = lString + self.lintString = lintString + self.mStringString = mStringString + self.attributes = attributes + self.unionField1 = unionField1 + self.unionField2 = unionField2 + self.unionField3 = unionField3 + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.aint = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.aString = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.lint = [] + (_etype19, _size16) = iprot.readListBegin() + for _i20 in xrange(_size16): + _elem21 = iprot.readI32(); + self.lint.append(_elem21) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.lString = [] + (_etype25, _size22) = iprot.readListBegin() + for _i26 in xrange(_size22): + _elem27 = iprot.readString(); + self.lString.append(_elem27) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.LIST: + self.lintString = [] + (_etype31, _size28) = iprot.readListBegin() + for _i32 in xrange(_size28): + _elem33 = IntString() + _elem33.read(iprot) + self.lintString.append(_elem33) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.MAP: + self.mStringString = {} + (_ktype35, _vtype36, _size34 ) = iprot.readMapBegin() + for _i38 in xrange(_size34): + _key39 = iprot.readString(); + _val40 = iprot.readString(); + self.mStringString[_key39] = _val40 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.MAP: + self.attributes = {} + (_ktype42, _vtype43, _size41 ) = iprot.readMapBegin() + for _i45 in xrange(_size41): + _key46 = iprot.readString(); + _val47 = {} + (_ktype49, _vtype50, _size48 ) = iprot.readMapBegin() + for _i52 in xrange(_size48): + _key53 = iprot.readString(); + _val54 = {} + (_ktype56, _vtype57, _size55 ) = iprot.readMapBegin() + for _i59 in xrange(_size55): + _key60 = iprot.readString(); + _val61 = PropValueUnion() + _val61.read(iprot) + _val54[_key60] = _val61 + iprot.readMapEnd() + _val47[_key53] = _val54 + iprot.readMapEnd() + self.attributes[_key46] = _val47 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRUCT: + self.unionField1 = PropValueUnion() + self.unionField1.read(iprot) + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.STRUCT: + self.unionField2 = PropValueUnion() + self.unionField2.read(iprot) + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.STRUCT: + self.unionField3 = PropValueUnion() + self.unionField3.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('Complex') + if self.aint is not None: + oprot.writeFieldBegin('aint', TType.I32, 1) + oprot.writeI32(self.aint) + oprot.writeFieldEnd() + if self.aString is not None: + oprot.writeFieldBegin('aString', TType.STRING, 2) + oprot.writeString(self.aString) + oprot.writeFieldEnd() + if self.lint is not None: + oprot.writeFieldBegin('lint', TType.LIST, 3) + oprot.writeListBegin(TType.I32, len(self.lint)) + for iter62 in self.lint: + oprot.writeI32(iter62) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.lString is not None: + oprot.writeFieldBegin('lString', TType.LIST, 4) + oprot.writeListBegin(TType.STRING, len(self.lString)) + for iter63 in self.lString: + oprot.writeString(iter63) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.lintString is not None: + oprot.writeFieldBegin('lintString', TType.LIST, 5) + oprot.writeListBegin(TType.STRUCT, len(self.lintString)) + for iter64 in self.lintString: + iter64.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.mStringString is not None: + oprot.writeFieldBegin('mStringString', TType.MAP, 6) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.mStringString)) + for kiter65,viter66 in self.mStringString.items(): + oprot.writeString(kiter65) + oprot.writeString(viter66) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.attributes is not None: + oprot.writeFieldBegin('attributes', TType.MAP, 7) + oprot.writeMapBegin(TType.STRING, TType.MAP, len(self.attributes)) + for kiter67,viter68 in self.attributes.items(): + oprot.writeString(kiter67) + oprot.writeMapBegin(TType.STRING, TType.MAP, len(viter68)) + for kiter69,viter70 in viter68.items(): + oprot.writeString(kiter69) + oprot.writeMapBegin(TType.STRING, TType.STRUCT, len(viter70)) + for kiter71,viter72 in viter70.items(): + oprot.writeString(kiter71) + viter72.write(oprot) + oprot.writeMapEnd() + oprot.writeMapEnd() + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.unionField1 is not None: + oprot.writeFieldBegin('unionField1', TType.STRUCT, 8) + self.unionField1.write(oprot) + oprot.writeFieldEnd() + if self.unionField2 is not None: + oprot.writeFieldBegin('unionField2', TType.STRUCT, 9) + self.unionField2.write(oprot) + oprot.writeFieldEnd() + if self.unionField3 is not None: + oprot.writeFieldBegin('unionField3', TType.STRUCT, 10) + self.unionField3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class SetIntString: + """ + Attributes: + - sIntString + - aString + """ + + thrift_spec = ( + None, # 0 + (1, TType.SET, 'sIntString', (TType.STRUCT,(IntString, IntString.thrift_spec)), None, ), # 1 + (2, TType.STRING, 'aString', None, None, ), # 2 + ) + + def __init__(self, sIntString=None, aString=None,): + self.sIntString = sIntString + self.aString = aString + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.SET: + self.sIntString = set() + (_etype76, _size73) = iprot.readSetBegin() + for _i77 in xrange(_size73): + _elem78 = IntString() + _elem78.read(iprot) + self.sIntString.add(_elem78) + iprot.readSetEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.aString = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('SetIntString') + if self.sIntString is not None: + oprot.writeFieldBegin('sIntString', TType.SET, 1) + oprot.writeSetBegin(TType.STRUCT, len(self.sIntString)) + for iter79 in self.sIntString: + iter79.write(oprot) + oprot.writeSetEnd() + oprot.writeFieldEnd() + if self.aString is not None: + oprot.writeFieldBegin('aString', TType.STRING, 2) + oprot.writeString(self.aString) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) diff --git a/storage-api/serde-api/src/gen/thrift/gen-py/megastruct/__init__.py b/storage-api/serde-api/src/gen/thrift/gen-py/megastruct/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..adefd8e51fcf17862843d83fd7ea34546ef09fca --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-py/megastruct/__init__.py @@ -0,0 +1 @@ +__all__ = ['ttypes', 'constants'] diff --git a/storage-api/serde-api/src/gen/thrift/gen-py/megastruct/constants.py b/storage-api/serde-api/src/gen/thrift/gen-py/megastruct/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..a236326cf38cf20a4d300bc6eb2eff0be93ef2e7 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-py/megastruct/constants.py @@ -0,0 +1,11 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TException, TApplicationException +from ttypes import * + diff --git a/storage-api/serde-api/src/gen/thrift/gen-py/megastruct/ttypes.py b/storage-api/serde-api/src/gen/thrift/gen-py/megastruct/ttypes.py new file mode 100644 index 0000000000000000000000000000000000000000..de516b246ac810fd7a25ca87b25b486abada4ad5 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-py/megastruct/ttypes.py @@ -0,0 +1,520 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TException, TApplicationException + +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol, TProtocol +try: + from thrift.protocol import fastbinary +except: + fastbinary = None + + +class MyEnum: + LLAMA = 1 + ALPACA = 2 + + _VALUES_TO_NAMES = { + 1: "LLAMA", + 2: "ALPACA", + } + + _NAMES_TO_VALUES = { + "LLAMA": 1, + "ALPACA": 2, + } + + +class MiniStruct: + """ + Attributes: + - my_string + - my_enum + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'my_string', None, None, ), # 1 + (2, TType.I32, 'my_enum', None, None, ), # 2 + ) + + def __init__(self, my_string=None, my_enum=None,): + self.my_string = my_string + self.my_enum = my_enum + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.my_string = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.my_enum = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('MiniStruct') + if self.my_string is not None: + oprot.writeFieldBegin('my_string', TType.STRING, 1) + oprot.writeString(self.my_string) + oprot.writeFieldEnd() + if self.my_enum is not None: + oprot.writeFieldBegin('my_enum', TType.I32, 2) + oprot.writeI32(self.my_enum) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class MegaStruct: + """ + Attributes: + - my_bool + - my_byte + - my_16bit_int + - my_32bit_int + - my_64bit_int + - my_double + - my_string + - my_binary + - my_string_string_map + - my_string_enum_map + - my_enum_string_map + - my_enum_struct_map + - my_enum_stringlist_map + - my_enum_structlist_map + - my_stringlist + - my_structlist + - my_enumlist + - my_stringset + - my_enumset + - my_structset + """ + + thrift_spec = ( + None, # 0 + (1, TType.BOOL, 'my_bool', None, None, ), # 1 + (2, TType.BYTE, 'my_byte', None, None, ), # 2 + (3, TType.I16, 'my_16bit_int', None, None, ), # 3 + (4, TType.I32, 'my_32bit_int', None, None, ), # 4 + (5, TType.I64, 'my_64bit_int', None, None, ), # 5 + (6, TType.DOUBLE, 'my_double', None, None, ), # 6 + (7, TType.STRING, 'my_string', None, None, ), # 7 + (8, TType.STRING, 'my_binary', None, None, ), # 8 + (9, TType.MAP, 'my_string_string_map', (TType.STRING,None,TType.STRING,None), None, ), # 9 + (10, TType.MAP, 'my_string_enum_map', (TType.STRING,None,TType.I32,None), None, ), # 10 + (11, TType.MAP, 'my_enum_string_map', (TType.I32,None,TType.STRING,None), None, ), # 11 + (12, TType.MAP, 'my_enum_struct_map', (TType.I32,None,TType.STRUCT,(MiniStruct, MiniStruct.thrift_spec)), None, ), # 12 + (13, TType.MAP, 'my_enum_stringlist_map', (TType.I32,None,TType.LIST,(TType.STRING,None)), None, ), # 13 + (14, TType.MAP, 'my_enum_structlist_map', (TType.I32,None,TType.LIST,(TType.STRUCT,(MiniStruct, MiniStruct.thrift_spec))), None, ), # 14 + (15, TType.LIST, 'my_stringlist', (TType.STRING,None), None, ), # 15 + (16, TType.LIST, 'my_structlist', (TType.STRUCT,(MiniStruct, MiniStruct.thrift_spec)), None, ), # 16 + (17, TType.LIST, 'my_enumlist', (TType.I32,None), None, ), # 17 + (18, TType.SET, 'my_stringset', (TType.STRING,None), None, ), # 18 + (19, TType.SET, 'my_enumset', (TType.I32,None), None, ), # 19 + (20, TType.SET, 'my_structset', (TType.STRUCT,(MiniStruct, MiniStruct.thrift_spec)), None, ), # 20 + ) + + def __init__(self, my_bool=None, my_byte=None, my_16bit_int=None, my_32bit_int=None, my_64bit_int=None, my_double=None, my_string=None, my_binary=None, my_string_string_map=None, my_string_enum_map=None, my_enum_string_map=None, my_enum_struct_map=None, my_enum_stringlist_map=None, my_enum_structlist_map=None, my_stringlist=None, my_structlist=None, my_enumlist=None, my_stringset=None, my_enumset=None, my_structset=None,): + self.my_bool = my_bool + self.my_byte = my_byte + self.my_16bit_int = my_16bit_int + self.my_32bit_int = my_32bit_int + self.my_64bit_int = my_64bit_int + self.my_double = my_double + self.my_string = my_string + self.my_binary = my_binary + self.my_string_string_map = my_string_string_map + self.my_string_enum_map = my_string_enum_map + self.my_enum_string_map = my_enum_string_map + self.my_enum_struct_map = my_enum_struct_map + self.my_enum_stringlist_map = my_enum_stringlist_map + self.my_enum_structlist_map = my_enum_structlist_map + self.my_stringlist = my_stringlist + self.my_structlist = my_structlist + self.my_enumlist = my_enumlist + self.my_stringset = my_stringset + self.my_enumset = my_enumset + self.my_structset = my_structset + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.BOOL: + self.my_bool = iprot.readBool(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BYTE: + self.my_byte = iprot.readByte(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I16: + self.my_16bit_int = iprot.readI16(); + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.my_32bit_int = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I64: + self.my_64bit_int = iprot.readI64(); + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.DOUBLE: + self.my_double = iprot.readDouble(); + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.my_string = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRING: + self.my_binary = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.MAP: + self.my_string_string_map = {} + (_ktype1, _vtype2, _size0 ) = iprot.readMapBegin() + for _i4 in xrange(_size0): + _key5 = iprot.readString(); + _val6 = iprot.readString(); + self.my_string_string_map[_key5] = _val6 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.MAP: + self.my_string_enum_map = {} + (_ktype8, _vtype9, _size7 ) = iprot.readMapBegin() + for _i11 in xrange(_size7): + _key12 = iprot.readString(); + _val13 = iprot.readI32(); + self.my_string_enum_map[_key12] = _val13 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 11: + if ftype == TType.MAP: + self.my_enum_string_map = {} + (_ktype15, _vtype16, _size14 ) = iprot.readMapBegin() + for _i18 in xrange(_size14): + _key19 = iprot.readI32(); + _val20 = iprot.readString(); + self.my_enum_string_map[_key19] = _val20 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 12: + if ftype == TType.MAP: + self.my_enum_struct_map = {} + (_ktype22, _vtype23, _size21 ) = iprot.readMapBegin() + for _i25 in xrange(_size21): + _key26 = iprot.readI32(); + _val27 = MiniStruct() + _val27.read(iprot) + self.my_enum_struct_map[_key26] = _val27 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 13: + if ftype == TType.MAP: + self.my_enum_stringlist_map = {} + (_ktype29, _vtype30, _size28 ) = iprot.readMapBegin() + for _i32 in xrange(_size28): + _key33 = iprot.readI32(); + _val34 = [] + (_etype38, _size35) = iprot.readListBegin() + for _i39 in xrange(_size35): + _elem40 = iprot.readString(); + _val34.append(_elem40) + iprot.readListEnd() + self.my_enum_stringlist_map[_key33] = _val34 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 14: + if ftype == TType.MAP: + self.my_enum_structlist_map = {} + (_ktype42, _vtype43, _size41 ) = iprot.readMapBegin() + for _i45 in xrange(_size41): + _key46 = iprot.readI32(); + _val47 = [] + (_etype51, _size48) = iprot.readListBegin() + for _i52 in xrange(_size48): + _elem53 = MiniStruct() + _elem53.read(iprot) + _val47.append(_elem53) + iprot.readListEnd() + self.my_enum_structlist_map[_key46] = _val47 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 15: + if ftype == TType.LIST: + self.my_stringlist = [] + (_etype57, _size54) = iprot.readListBegin() + for _i58 in xrange(_size54): + _elem59 = iprot.readString(); + self.my_stringlist.append(_elem59) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 16: + if ftype == TType.LIST: + self.my_structlist = [] + (_etype63, _size60) = iprot.readListBegin() + for _i64 in xrange(_size60): + _elem65 = MiniStruct() + _elem65.read(iprot) + self.my_structlist.append(_elem65) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 17: + if ftype == TType.LIST: + self.my_enumlist = [] + (_etype69, _size66) = iprot.readListBegin() + for _i70 in xrange(_size66): + _elem71 = iprot.readI32(); + self.my_enumlist.append(_elem71) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 18: + if ftype == TType.SET: + self.my_stringset = set() + (_etype75, _size72) = iprot.readSetBegin() + for _i76 in xrange(_size72): + _elem77 = iprot.readString(); + self.my_stringset.add(_elem77) + iprot.readSetEnd() + else: + iprot.skip(ftype) + elif fid == 19: + if ftype == TType.SET: + self.my_enumset = set() + (_etype81, _size78) = iprot.readSetBegin() + for _i82 in xrange(_size78): + _elem83 = iprot.readI32(); + self.my_enumset.add(_elem83) + iprot.readSetEnd() + else: + iprot.skip(ftype) + elif fid == 20: + if ftype == TType.SET: + self.my_structset = set() + (_etype87, _size84) = iprot.readSetBegin() + for _i88 in xrange(_size84): + _elem89 = MiniStruct() + _elem89.read(iprot) + self.my_structset.add(_elem89) + iprot.readSetEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('MegaStruct') + if self.my_bool is not None: + oprot.writeFieldBegin('my_bool', TType.BOOL, 1) + oprot.writeBool(self.my_bool) + oprot.writeFieldEnd() + if self.my_byte is not None: + oprot.writeFieldBegin('my_byte', TType.BYTE, 2) + oprot.writeByte(self.my_byte) + oprot.writeFieldEnd() + if self.my_16bit_int is not None: + oprot.writeFieldBegin('my_16bit_int', TType.I16, 3) + oprot.writeI16(self.my_16bit_int) + oprot.writeFieldEnd() + if self.my_32bit_int is not None: + oprot.writeFieldBegin('my_32bit_int', TType.I32, 4) + oprot.writeI32(self.my_32bit_int) + oprot.writeFieldEnd() + if self.my_64bit_int is not None: + oprot.writeFieldBegin('my_64bit_int', TType.I64, 5) + oprot.writeI64(self.my_64bit_int) + oprot.writeFieldEnd() + if self.my_double is not None: + oprot.writeFieldBegin('my_double', TType.DOUBLE, 6) + oprot.writeDouble(self.my_double) + oprot.writeFieldEnd() + if self.my_string is not None: + oprot.writeFieldBegin('my_string', TType.STRING, 7) + oprot.writeString(self.my_string) + oprot.writeFieldEnd() + if self.my_binary is not None: + oprot.writeFieldBegin('my_binary', TType.STRING, 8) + oprot.writeString(self.my_binary) + oprot.writeFieldEnd() + if self.my_string_string_map is not None: + oprot.writeFieldBegin('my_string_string_map', TType.MAP, 9) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.my_string_string_map)) + for kiter90,viter91 in self.my_string_string_map.items(): + oprot.writeString(kiter90) + oprot.writeString(viter91) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.my_string_enum_map is not None: + oprot.writeFieldBegin('my_string_enum_map', TType.MAP, 10) + oprot.writeMapBegin(TType.STRING, TType.I32, len(self.my_string_enum_map)) + for kiter92,viter93 in self.my_string_enum_map.items(): + oprot.writeString(kiter92) + oprot.writeI32(viter93) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.my_enum_string_map is not None: + oprot.writeFieldBegin('my_enum_string_map', TType.MAP, 11) + oprot.writeMapBegin(TType.I32, TType.STRING, len(self.my_enum_string_map)) + for kiter94,viter95 in self.my_enum_string_map.items(): + oprot.writeI32(kiter94) + oprot.writeString(viter95) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.my_enum_struct_map is not None: + oprot.writeFieldBegin('my_enum_struct_map', TType.MAP, 12) + oprot.writeMapBegin(TType.I32, TType.STRUCT, len(self.my_enum_struct_map)) + for kiter96,viter97 in self.my_enum_struct_map.items(): + oprot.writeI32(kiter96) + viter97.write(oprot) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.my_enum_stringlist_map is not None: + oprot.writeFieldBegin('my_enum_stringlist_map', TType.MAP, 13) + oprot.writeMapBegin(TType.I32, TType.LIST, len(self.my_enum_stringlist_map)) + for kiter98,viter99 in self.my_enum_stringlist_map.items(): + oprot.writeI32(kiter98) + oprot.writeListBegin(TType.STRING, len(viter99)) + for iter100 in viter99: + oprot.writeString(iter100) + oprot.writeListEnd() + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.my_enum_structlist_map is not None: + oprot.writeFieldBegin('my_enum_structlist_map', TType.MAP, 14) + oprot.writeMapBegin(TType.I32, TType.LIST, len(self.my_enum_structlist_map)) + for kiter101,viter102 in self.my_enum_structlist_map.items(): + oprot.writeI32(kiter101) + oprot.writeListBegin(TType.STRUCT, len(viter102)) + for iter103 in viter102: + iter103.write(oprot) + oprot.writeListEnd() + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.my_stringlist is not None: + oprot.writeFieldBegin('my_stringlist', TType.LIST, 15) + oprot.writeListBegin(TType.STRING, len(self.my_stringlist)) + for iter104 in self.my_stringlist: + oprot.writeString(iter104) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.my_structlist is not None: + oprot.writeFieldBegin('my_structlist', TType.LIST, 16) + oprot.writeListBegin(TType.STRUCT, len(self.my_structlist)) + for iter105 in self.my_structlist: + iter105.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.my_enumlist is not None: + oprot.writeFieldBegin('my_enumlist', TType.LIST, 17) + oprot.writeListBegin(TType.I32, len(self.my_enumlist)) + for iter106 in self.my_enumlist: + oprot.writeI32(iter106) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.my_stringset is not None: + oprot.writeFieldBegin('my_stringset', TType.SET, 18) + oprot.writeSetBegin(TType.STRING, len(self.my_stringset)) + for iter107 in self.my_stringset: + oprot.writeString(iter107) + oprot.writeSetEnd() + oprot.writeFieldEnd() + if self.my_enumset is not None: + oprot.writeFieldBegin('my_enumset', TType.SET, 19) + oprot.writeSetBegin(TType.I32, len(self.my_enumset)) + for iter108 in self.my_enumset: + oprot.writeI32(iter108) + oprot.writeSetEnd() + oprot.writeFieldEnd() + if self.my_structset is not None: + oprot.writeFieldBegin('my_structset', TType.SET, 20) + oprot.writeSetBegin(TType.STRUCT, len(self.my_structset)) + for iter109 in self.my_structset: + iter109.write(oprot) + oprot.writeSetEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) diff --git a/storage-api/serde-api/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/__init__.py b/storage-api/serde-api/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..adefd8e51fcf17862843d83fd7ea34546ef09fca --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/__init__.py @@ -0,0 +1 @@ +__all__ = ['ttypes', 'constants'] diff --git a/storage-api/serde-api/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py b/storage-api/serde-api/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..aef2525173e1c2f8389d04cf5a942f60fff93784 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/constants.py @@ -0,0 +1,88 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TException, TApplicationException +from ttypes import * + +SERIALIZATION_LIB = "serialization.lib" +SERIALIZATION_CLASS = "serialization.class" +SERIALIZATION_FORMAT = "serialization.format" +SERIALIZATION_DDL = "serialization.ddl" +SERIALIZATION_NULL_FORMAT = "serialization.null.format" +SERIALIZATION_ESCAPE_CRLF = "serialization.escape.crlf" +SERIALIZATION_LAST_COLUMN_TAKES_REST = "serialization.last.column.takes.rest" +SERIALIZATION_SORT_ORDER = "serialization.sort.order" +SERIALIZATION_NULL_SORT_ORDER = "serialization.sort.order.null" +SERIALIZATION_USE_JSON_OBJECTS = "serialization.use.json.object" +SERIALIZATION_ENCODING = "serialization.encoding" +FIELD_DELIM = "field.delim" +COLLECTION_DELIM = "collection.delim" +LINE_DELIM = "line.delim" +MAPKEY_DELIM = "mapkey.delim" +QUOTE_CHAR = "quote.delim" +ESCAPE_CHAR = "escape.delim" +HEADER_COUNT = "skip.header.line.count" +FOOTER_COUNT = "skip.footer.line.count" +VOID_TYPE_NAME = "void" +BOOLEAN_TYPE_NAME = "boolean" +TINYINT_TYPE_NAME = "tinyint" +SMALLINT_TYPE_NAME = "smallint" +INT_TYPE_NAME = "int" +BIGINT_TYPE_NAME = "bigint" +FLOAT_TYPE_NAME = "float" +DOUBLE_TYPE_NAME = "double" +STRING_TYPE_NAME = "string" +CHAR_TYPE_NAME = "char" +VARCHAR_TYPE_NAME = "varchar" +DATE_TYPE_NAME = "date" +DATETIME_TYPE_NAME = "datetime" +TIMESTAMP_TYPE_NAME = "timestamp" +DECIMAL_TYPE_NAME = "decimal" +BINARY_TYPE_NAME = "binary" +INTERVAL_YEAR_MONTH_TYPE_NAME = "interval_year_month" +INTERVAL_DAY_TIME_TYPE_NAME = "interval_day_time" +TIMESTAMPLOCALTZ_TYPE_NAME = "timestamp with local time zone" +LIST_TYPE_NAME = "array" +MAP_TYPE_NAME = "map" +STRUCT_TYPE_NAME = "struct" +UNION_TYPE_NAME = "uniontype" +LIST_COLUMNS = "columns" +LIST_COLUMN_TYPES = "columns.types" +TIMESTAMP_FORMATS = "timestamp.formats" +COLUMN_NAME_DELIMITER = "column.name.delimiter" +PrimitiveTypes = set([ + "void", + "boolean", + "tinyint", + "smallint", + "int", + "bigint", + "float", + "double", + "string", + "varchar", + "char", + "date", + "datetime", + "timestamp", + "interval_year_month", + "interval_day_time", + "decimal", + "binary", + "timestamp with local time zone", +]) +CollectionTypes = set([ + "array", + "map", +]) +IntegralTypes = set([ + "tinyint", + "smallint", + "int", + "bigint", +]) diff --git a/storage-api/serde-api/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/ttypes.py b/storage-api/serde-api/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/ttypes.py new file mode 100644 index 0000000000000000000000000000000000000000..ad3e28051bfb8d1b327a8761a26370f4f945c0d7 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-py/org_apache_hadoop_hive_serde/ttypes.py @@ -0,0 +1,18 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TException, TApplicationException + +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol, TProtocol +try: + from thrift.protocol import fastbinary +except: + fastbinary = None + + diff --git a/storage-api/serde-api/src/gen/thrift/gen-py/testthrift/__init__.py b/storage-api/serde-api/src/gen/thrift/gen-py/testthrift/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..adefd8e51fcf17862843d83fd7ea34546ef09fca --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-py/testthrift/__init__.py @@ -0,0 +1 @@ +__all__ = ['ttypes', 'constants'] diff --git a/storage-api/serde-api/src/gen/thrift/gen-py/testthrift/constants.py b/storage-api/serde-api/src/gen/thrift/gen-py/testthrift/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..a236326cf38cf20a4d300bc6eb2eff0be93ef2e7 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-py/testthrift/constants.py @@ -0,0 +1,11 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TException, TApplicationException +from ttypes import * + diff --git a/storage-api/serde-api/src/gen/thrift/gen-py/testthrift/ttypes.py b/storage-api/serde-api/src/gen/thrift/gen-py/testthrift/ttypes.py new file mode 100644 index 0000000000000000000000000000000000000000..a8b2224e2e46e33c1531c351da84da30c7ac8fd7 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-py/testthrift/ttypes.py @@ -0,0 +1,171 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TException, TApplicationException + +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol, TProtocol +try: + from thrift.protocol import fastbinary +except: + fastbinary = None + + + +class InnerStruct: + """ + Attributes: + - field0 + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'field0', None, None, ), # 1 + ) + + def __init__(self, field0=None,): + self.field0 = field0 + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.field0 = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('InnerStruct') + if self.field0 is not None: + oprot.writeFieldBegin('field0', TType.I32, 1) + oprot.writeI32(self.field0) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class ThriftTestObj: + """ + Attributes: + - field1 + - field2 + - field3 + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'field1', None, None, ), # 1 + (2, TType.STRING, 'field2', None, None, ), # 2 + (3, TType.LIST, 'field3', (TType.STRUCT,(InnerStruct, InnerStruct.thrift_spec)), None, ), # 3 + ) + + def __init__(self, field1=None, field2=None, field3=None,): + self.field1 = field1 + self.field2 = field2 + self.field3 = field3 + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.field1 = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.field2 = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.field3 = [] + (_etype3, _size0) = iprot.readListBegin() + for _i4 in xrange(_size0): + _elem5 = InnerStruct() + _elem5.read(iprot) + self.field3.append(_elem5) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('ThriftTestObj') + if self.field1 is not None: + oprot.writeFieldBegin('field1', TType.I32, 1) + oprot.writeI32(self.field1) + oprot.writeFieldEnd() + if self.field2 is not None: + oprot.writeFieldBegin('field2', TType.STRING, 2) + oprot.writeString(self.field2) + oprot.writeFieldEnd() + if self.field3 is not None: + oprot.writeFieldBegin('field3', TType.LIST, 3) + oprot.writeListBegin(TType.STRUCT, len(self.field3)) + for iter6 in self.field3: + iter6.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) diff --git a/storage-api/serde-api/src/gen/thrift/gen-rb/complex_constants.rb b/storage-api/serde-api/src/gen/thrift/gen-rb/complex_constants.rb new file mode 100644 index 0000000000000000000000000000000000000000..00ee2e8e86e8a913c67227b66714cd8093640ca5 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-rb/complex_constants.rb @@ -0,0 +1,9 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# + +require 'thrift' +require 'complex_types' + diff --git a/storage-api/serde-api/src/gen/thrift/gen-rb/complex_types.rb b/storage-api/serde-api/src/gen/thrift/gen-rb/complex_types.rb new file mode 100644 index 0000000000000000000000000000000000000000..ffbd4d7c3a3c4f540e225adaaeff3d801ab18855 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-rb/complex_types.rb @@ -0,0 +1,139 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# + +require 'thrift' + +class PropValueUnion < ::Thrift::Union + include ::Thrift::Struct_Union + class << self + def intValue(val) + PropValueUnion.new(:intValue, val) + end + + def longValue(val) + PropValueUnion.new(:longValue, val) + end + + def stringValue(val) + PropValueUnion.new(:stringValue, val) + end + + def doubleValue(val) + PropValueUnion.new(:doubleValue, val) + end + + def flag(val) + PropValueUnion.new(:flag, val) + end + + def lString(val) + PropValueUnion.new(:lString, val) + end + + def unionMStringString(val) + PropValueUnion.new(:unionMStringString, val) + end + end + + INTVALUE = 1 + LONGVALUE = 2 + STRINGVALUE = 3 + DOUBLEVALUE = 4 + FLAG = 5 + LSTRING = 6 + UNIONMSTRINGSTRING = 7 + + FIELDS = { + INTVALUE => {:type => ::Thrift::Types::I32, :name => 'intValue', :optional => true}, + LONGVALUE => {:type => ::Thrift::Types::I64, :name => 'longValue', :optional => true}, + STRINGVALUE => {:type => ::Thrift::Types::STRING, :name => 'stringValue', :optional => true}, + DOUBLEVALUE => {:type => ::Thrift::Types::DOUBLE, :name => 'doubleValue', :optional => true}, + FLAG => {:type => ::Thrift::Types::BOOL, :name => 'flag', :optional => true}, + LSTRING => {:type => ::Thrift::Types::LIST, :name => 'lString', :element => {:type => ::Thrift::Types::STRING}}, + UNIONMSTRINGSTRING => {:type => ::Thrift::Types::MAP, :name => 'unionMStringString', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}} + } + + def struct_fields; FIELDS; end + + def validate + raise(StandardError, 'Union fields are not set.') if get_set_field.nil? || get_value.nil? + end + + ::Thrift::Union.generate_accessors self +end + +class IntString + include ::Thrift::Struct, ::Thrift::Struct_Union + MYINT = 1 + MYSTRING = 2 + UNDERSCORE_INT = 3 + + FIELDS = { + MYINT => {:type => ::Thrift::Types::I32, :name => 'myint'}, + MYSTRING => {:type => ::Thrift::Types::STRING, :name => 'myString'}, + UNDERSCORE_INT => {:type => ::Thrift::Types::I32, :name => 'underscore_int'} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self +end + +class Complex + include ::Thrift::Struct, ::Thrift::Struct_Union + AINT = 1 + ASTRING = 2 + LINT = 3 + LSTRING = 4 + LINTSTRING = 5 + MSTRINGSTRING = 6 + ATTRIBUTES = 7 + UNIONFIELD1 = 8 + UNIONFIELD2 = 9 + UNIONFIELD3 = 10 + + FIELDS = { + AINT => {:type => ::Thrift::Types::I32, :name => 'aint'}, + ASTRING => {:type => ::Thrift::Types::STRING, :name => 'aString'}, + LINT => {:type => ::Thrift::Types::LIST, :name => 'lint', :element => {:type => ::Thrift::Types::I32}}, + LSTRING => {:type => ::Thrift::Types::LIST, :name => 'lString', :element => {:type => ::Thrift::Types::STRING}}, + LINTSTRING => {:type => ::Thrift::Types::LIST, :name => 'lintString', :element => {:type => ::Thrift::Types::STRUCT, :class => ::IntString}}, + MSTRINGSTRING => {:type => ::Thrift::Types::MAP, :name => 'mStringString', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}}, + ATTRIBUTES => {:type => ::Thrift::Types::MAP, :name => 'attributes', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::MAP, :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::MAP, :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRUCT, :class => ::PropValueUnion}}}}, + UNIONFIELD1 => {:type => ::Thrift::Types::STRUCT, :name => 'unionField1', :class => ::PropValueUnion}, + UNIONFIELD2 => {:type => ::Thrift::Types::STRUCT, :name => 'unionField2', :class => ::PropValueUnion}, + UNIONFIELD3 => {:type => ::Thrift::Types::STRUCT, :name => 'unionField3', :class => ::PropValueUnion} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self +end + +class SetIntString + include ::Thrift::Struct, ::Thrift::Struct_Union + SINTSTRING = 1 + ASTRING = 2 + + FIELDS = { + SINTSTRING => {:type => ::Thrift::Types::SET, :name => 'sIntString', :element => {:type => ::Thrift::Types::STRUCT, :class => ::IntString}}, + ASTRING => {:type => ::Thrift::Types::STRING, :name => 'aString'} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self +end + diff --git a/storage-api/serde-api/src/gen/thrift/gen-rb/megastruct_constants.rb b/storage-api/serde-api/src/gen/thrift/gen-rb/megastruct_constants.rb new file mode 100644 index 0000000000000000000000000000000000000000..91193eb4315e199cdcf3f1601b4426b1469230e3 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-rb/megastruct_constants.rb @@ -0,0 +1,9 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# + +require 'thrift' +require 'megastruct_types' + diff --git a/storage-api/serde-api/src/gen/thrift/gen-rb/megastruct_types.rb b/storage-api/serde-api/src/gen/thrift/gen-rb/megastruct_types.rb new file mode 100644 index 0000000000000000000000000000000000000000..c299cf35d4a857bda434a29d508c7ec4afe91de9 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-rb/megastruct_types.rb @@ -0,0 +1,90 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# + +require 'thrift' + +module MyEnum + LLAMA = 1 + ALPACA = 2 + VALUE_MAP = {1 => "LLAMA", 2 => "ALPACA"} + VALID_VALUES = Set.new([LLAMA, ALPACA]).freeze +end + +class MiniStruct + include ::Thrift::Struct, ::Thrift::Struct_Union + MY_STRING = 1 + MY_ENUM = 2 + + FIELDS = { + MY_STRING => {:type => ::Thrift::Types::STRING, :name => 'my_string', :optional => true}, + MY_ENUM => {:type => ::Thrift::Types::I32, :name => 'my_enum', :optional => true, :enum_class => ::MyEnum} + } + + def struct_fields; FIELDS; end + + def validate + unless @my_enum.nil? || ::MyEnum::VALID_VALUES.include?(@my_enum) + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field my_enum!') + end + end + + ::Thrift::Struct.generate_accessors self +end + +class MegaStruct + include ::Thrift::Struct, ::Thrift::Struct_Union + MY_BOOL = 1 + MY_BYTE = 2 + MY_16BIT_INT = 3 + MY_32BIT_INT = 4 + MY_64BIT_INT = 5 + MY_DOUBLE = 6 + MY_STRING = 7 + MY_BINARY = 8 + MY_STRING_STRING_MAP = 9 + MY_STRING_ENUM_MAP = 10 + MY_ENUM_STRING_MAP = 11 + MY_ENUM_STRUCT_MAP = 12 + MY_ENUM_STRINGLIST_MAP = 13 + MY_ENUM_STRUCTLIST_MAP = 14 + MY_STRINGLIST = 15 + MY_STRUCTLIST = 16 + MY_ENUMLIST = 17 + MY_STRINGSET = 18 + MY_ENUMSET = 19 + MY_STRUCTSET = 20 + + FIELDS = { + MY_BOOL => {:type => ::Thrift::Types::BOOL, :name => 'my_bool', :optional => true}, + MY_BYTE => {:type => ::Thrift::Types::BYTE, :name => 'my_byte', :optional => true}, + MY_16BIT_INT => {:type => ::Thrift::Types::I16, :name => 'my_16bit_int', :optional => true}, + MY_32BIT_INT => {:type => ::Thrift::Types::I32, :name => 'my_32bit_int', :optional => true}, + MY_64BIT_INT => {:type => ::Thrift::Types::I64, :name => 'my_64bit_int', :optional => true}, + MY_DOUBLE => {:type => ::Thrift::Types::DOUBLE, :name => 'my_double', :optional => true}, + MY_STRING => {:type => ::Thrift::Types::STRING, :name => 'my_string', :optional => true}, + MY_BINARY => {:type => ::Thrift::Types::STRING, :name => 'my_binary', :binary => true, :optional => true}, + MY_STRING_STRING_MAP => {:type => ::Thrift::Types::MAP, :name => 'my_string_string_map', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}, :optional => true}, + MY_STRING_ENUM_MAP => {:type => ::Thrift::Types::MAP, :name => 'my_string_enum_map', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::I32, :enum_class => ::MyEnum}, :optional => true}, + MY_ENUM_STRING_MAP => {:type => ::Thrift::Types::MAP, :name => 'my_enum_string_map', :key => {:type => ::Thrift::Types::I32, :enum_class => ::MyEnum}, :value => {:type => ::Thrift::Types::STRING}, :optional => true}, + MY_ENUM_STRUCT_MAP => {:type => ::Thrift::Types::MAP, :name => 'my_enum_struct_map', :key => {:type => ::Thrift::Types::I32, :enum_class => ::MyEnum}, :value => {:type => ::Thrift::Types::STRUCT, :class => ::MiniStruct}, :optional => true}, + MY_ENUM_STRINGLIST_MAP => {:type => ::Thrift::Types::MAP, :name => 'my_enum_stringlist_map', :key => {:type => ::Thrift::Types::I32, :enum_class => ::MyEnum}, :value => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRING}}, :optional => true}, + MY_ENUM_STRUCTLIST_MAP => {:type => ::Thrift::Types::MAP, :name => 'my_enum_structlist_map', :key => {:type => ::Thrift::Types::I32, :enum_class => ::MyEnum}, :value => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRUCT, :class => ::MiniStruct}}, :optional => true}, + MY_STRINGLIST => {:type => ::Thrift::Types::LIST, :name => 'my_stringlist', :element => {:type => ::Thrift::Types::STRING}, :optional => true}, + MY_STRUCTLIST => {:type => ::Thrift::Types::LIST, :name => 'my_structlist', :element => {:type => ::Thrift::Types::STRUCT, :class => ::MiniStruct}, :optional => true}, + MY_ENUMLIST => {:type => ::Thrift::Types::LIST, :name => 'my_enumlist', :element => {:type => ::Thrift::Types::I32, :enum_class => ::MyEnum}, :optional => true}, + MY_STRINGSET => {:type => ::Thrift::Types::SET, :name => 'my_stringset', :element => {:type => ::Thrift::Types::STRING}, :optional => true}, + MY_ENUMSET => {:type => ::Thrift::Types::SET, :name => 'my_enumset', :element => {:type => ::Thrift::Types::I32, :enum_class => ::MyEnum}, :optional => true}, + MY_STRUCTSET => {:type => ::Thrift::Types::SET, :name => 'my_structset', :element => {:type => ::Thrift::Types::STRUCT, :class => ::MiniStruct}, :optional => true} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self +end + diff --git a/storage-api/serde-api/src/gen/thrift/gen-rb/serde_constants.rb b/storage-api/serde-api/src/gen/thrift/gen-rb/serde_constants.rb new file mode 100644 index 0000000000000000000000000000000000000000..43ee2601d65c0fd7360dadb8a98f5277022c204c --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-rb/serde_constants.rb @@ -0,0 +1,135 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# + +require 'thrift' +require 'serde_types' + +SERIALIZATION_LIB = %q"serialization.lib" + +SERIALIZATION_CLASS = %q"serialization.class" + +SERIALIZATION_FORMAT = %q"serialization.format" + +SERIALIZATION_DDL = %q"serialization.ddl" + +SERIALIZATION_NULL_FORMAT = %q"serialization.null.format" + +SERIALIZATION_ESCAPE_CRLF = %q"serialization.escape.crlf" + +SERIALIZATION_LAST_COLUMN_TAKES_REST = %q"serialization.last.column.takes.rest" + +SERIALIZATION_SORT_ORDER = %q"serialization.sort.order" + +SERIALIZATION_NULL_SORT_ORDER = %q"serialization.sort.order.null" + +SERIALIZATION_USE_JSON_OBJECTS = %q"serialization.use.json.object" + +SERIALIZATION_ENCODING = %q"serialization.encoding" + +FIELD_DELIM = %q"field.delim" + +COLLECTION_DELIM = %q"collection.delim" + +LINE_DELIM = %q"line.delim" + +MAPKEY_DELIM = %q"mapkey.delim" + +QUOTE_CHAR = %q"quote.delim" + +ESCAPE_CHAR = %q"escape.delim" + +HEADER_COUNT = %q"skip.header.line.count" + +FOOTER_COUNT = %q"skip.footer.line.count" + +VOID_TYPE_NAME = %q"void" + +BOOLEAN_TYPE_NAME = %q"boolean" + +TINYINT_TYPE_NAME = %q"tinyint" + +SMALLINT_TYPE_NAME = %q"smallint" + +INT_TYPE_NAME = %q"int" + +BIGINT_TYPE_NAME = %q"bigint" + +FLOAT_TYPE_NAME = %q"float" + +DOUBLE_TYPE_NAME = %q"double" + +STRING_TYPE_NAME = %q"string" + +CHAR_TYPE_NAME = %q"char" + +VARCHAR_TYPE_NAME = %q"varchar" + +DATE_TYPE_NAME = %q"date" + +DATETIME_TYPE_NAME = %q"datetime" + +TIMESTAMP_TYPE_NAME = %q"timestamp" + +DECIMAL_TYPE_NAME = %q"decimal" + +BINARY_TYPE_NAME = %q"binary" + +INTERVAL_YEAR_MONTH_TYPE_NAME = %q"interval_year_month" + +INTERVAL_DAY_TIME_TYPE_NAME = %q"interval_day_time" + +TIMESTAMPLOCALTZ_TYPE_NAME = %q"timestamp with local time zone" + +LIST_TYPE_NAME = %q"array" + +MAP_TYPE_NAME = %q"map" + +STRUCT_TYPE_NAME = %q"struct" + +UNION_TYPE_NAME = %q"uniontype" + +LIST_COLUMNS = %q"columns" + +LIST_COLUMN_TYPES = %q"columns.types" + +TIMESTAMP_FORMATS = %q"timestamp.formats" + +COLUMN_NAME_DELIMITER = %q"column.name.delimiter" + +PrimitiveTypes = Set.new([ + %q"void", + %q"boolean", + %q"tinyint", + %q"smallint", + %q"int", + %q"bigint", + %q"float", + %q"double", + %q"string", + %q"varchar", + %q"char", + %q"date", + %q"datetime", + %q"timestamp", + %q"interval_year_month", + %q"interval_day_time", + %q"decimal", + %q"binary", + %q"timestamp with local time zone", +]) + +CollectionTypes = Set.new([ + %q"array", + %q"map", +]) + +IntegralTypes = Set.new([ + %q"tinyint", + %q"smallint", + %q"int", + %q"bigint", +]) + diff --git a/storage-api/serde-api/src/gen/thrift/gen-rb/serde_types.rb b/storage-api/serde-api/src/gen/thrift/gen-rb/serde_types.rb new file mode 100644 index 0000000000000000000000000000000000000000..d4260478bc10c80c1e0253c3fbcc8f6d78b880e8 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-rb/serde_types.rb @@ -0,0 +1,8 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# + +require 'thrift' + diff --git a/storage-api/serde-api/src/gen/thrift/gen-rb/testthrift_constants.rb b/storage-api/serde-api/src/gen/thrift/gen-rb/testthrift_constants.rb new file mode 100644 index 0000000000000000000000000000000000000000..13616eecfa5069d2472c559da8b63cb4ca229892 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-rb/testthrift_constants.rb @@ -0,0 +1,9 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# + +require 'thrift' +require 'testthrift_types' + diff --git a/storage-api/serde-api/src/gen/thrift/gen-rb/testthrift_types.rb b/storage-api/serde-api/src/gen/thrift/gen-rb/testthrift_types.rb new file mode 100644 index 0000000000000000000000000000000000000000..3f76ae61a7a0de6ac726299b4234601213f986a5 --- /dev/null +++ b/storage-api/serde-api/src/gen/thrift/gen-rb/testthrift_types.rb @@ -0,0 +1,44 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# + +require 'thrift' + +class InnerStruct + include ::Thrift::Struct, ::Thrift::Struct_Union + FIELD0 = 1 + + FIELDS = { + FIELD0 => {:type => ::Thrift::Types::I32, :name => 'field0'} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self +end + +class ThriftTestObj + include ::Thrift::Struct, ::Thrift::Struct_Union + FIELD1 = 1 + FIELD2 = 2 + FIELD3 = 3 + + FIELDS = { + FIELD1 => {:type => ::Thrift::Types::I32, :name => 'field1'}, + FIELD2 => {:type => ::Thrift::Types::STRING, :name => 'field2'}, + FIELD3 => {:type => ::Thrift::Types::LIST, :name => 'field3', :element => {:type => ::Thrift::Types::STRUCT, :class => ::InnerStruct}} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self +end + diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java new file mode 100644 index 0000000000000000000000000000000000000000..4b678f7e93b0c25fd87454fc44f7a83e1692fe3a --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2; + +import java.util.Map; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.io.Writable; + +import javax.annotation.Nullable; + +/** + * Abstract class for implementing SerDe. The abstract class has been created, so that + * new methods can be added in the underlying interface, SerDe, and only implementations + * that need those methods overwrite it. + */ +public abstract class AbstractSerDe implements Deserializer, Serializer { + + protected String configErrors; + + /** + * Initialize the SerDe. By default, this will use one set of properties, either the + * table properties or the partition properties. If a SerDe needs access to both sets, + * it should override this method. + * + * Eventually, once all SerDes have implemented this method, + * we should convert it to an abstract method. + * + * @param configuration Hadoop configuration + * @param tableProperties Table properties + * @param partitionProperties Partition properties + * @throws SerDeException + */ + public void initialize(Configuration configuration, Properties tableProperties, + Properties partitionProperties) throws SerDeException { + initialize(configuration, + createOverlayedProperties(tableProperties, partitionProperties)); + } + + /** + * Initialize the HiveSerializer. + * + * @param conf + * System properties. Can be null in compile time + * @param tbl + * table properties + * @throws SerDeException + */ + @Deprecated + public abstract void initialize(@Nullable Configuration conf, Properties tbl) + throws SerDeException; + + /** + * Returns the Writable class that would be returned by the serialize method. + * This is used to initialize SequenceFile header. + */ + public abstract Class getSerializedClass(); + + /** + * Serialize an object by navigating inside the Object with the + * ObjectInspector. In most cases, the return value of this function will be + * constant since the function will reuse the Writable object. If the client + * wants to keep a copy of the Writable, the client needs to clone the + * returned value. + */ + public abstract Writable serialize(Object obj, ObjectInspector objInspector) + throws SerDeException; + + /** + * Returns statistics collected when serializing + */ + public abstract SerDeStats getSerDeStats(); + + /** + * Deserialize an object out of a Writable blob. In most cases, the return + * value of this function will be constant since the function will reuse the + * returned object. If the client wants to keep a copy of the object, the + * client needs to clone the returned value by calling + * ObjectInspectorUtils.getStandardObject(). + * + * @param blob + * The Writable object containing a serialized object + * @return A Java object representing the contents in the blob. + */ + public abstract Object deserialize(Writable blob) throws SerDeException; + + /** + * Get the object inspector that can be used to navigate through the internal + * structure of the Object returned from deserialize(...). + */ + public abstract ObjectInspector getObjectInspector() throws SerDeException; + + /** + * Get the error messages during the Serde configuration + * + * @return The error messages in the configuration which are empty if no error occurred + */ + public String getConfigurationErrors() { + return configErrors == null ? "" : configErrors; + } + + /** + * @return Whether the SerDe that can store schema both inside and outside of metastore + * does, in fact, store it inside metastore, based on table parameters. + */ + public boolean shouldStoreFieldsInMetastore(Map tableParams) { + return false; // The default, unless SerDe overrides it. + } + + /** + * Returns the union of table and partition properties, + * with partition properties taking precedence. + * @param tblProps + * @param partProps + * @return the overlayed properties + */ + private static Properties createOverlayedProperties(Properties tblProps, Properties partProps) { + Properties props = new Properties(); + props.putAll(tblProps); + if (partProps != null) { + props.putAll(partProps); + } + return props; + } +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/Deserializer.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/Deserializer.java new file mode 100644 index 0000000000000000000000000000000000000000..a1d3dd87665e2358209d92a4f9efcbe5c57418ea --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/Deserializer.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2; + +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.io.Writable; + +/** + * HiveDeserializer is used to deserialize the data from hadoop Writable to a + * custom java object that can be of any type that the developer wants. + * + * HiveDeserializer also provides the ObjectInspector which can be used to + * inspect the internal structure of the object (that is returned by deserialize + * function). + * All deserializers should extend the abstract class AbstractDeserializer. + * The interface is necessary for SerDes to be able to implement both Serializer and Deserializer. + */ +public interface Deserializer { + + /** + * Initialize the HiveDeserializer. + * + * @param conf + * System properties + * @param tbl + * table properties + * @throws SerDeException + */ + void initialize(Configuration conf, Properties tbl) throws SerDeException; + + /** + * Deserialize an object out of a Writable blob. In most cases, the return + * value of this function will be constant since the function will reuse the + * returned object. If the client wants to keep a copy of the object, the + * client needs to clone the returned deserialized value by calling + * ObjectInspectorUtils.getStandardObject(). + * + * @param blob + * The Writable object containing a serialized object + * @return A Java object representing the contents in the blob. + */ + Object deserialize(Writable blob) throws SerDeException; + + /** + * Get the object inspector that can be used to navigate through the internal + * structure of the Object returned from deserialize(...). + */ + ObjectInspector getObjectInspector() throws SerDeException; + + /** + * Returns statistics collected when serializing + */ + SerDeStats getSerDeStats(); +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/SerDeException.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/SerDeException.java new file mode 100644 index 0000000000000000000000000000000000000000..ea1ae9ca006754a19c0ab51d4d02be125476ee7e --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/SerDeException.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2; + +/** + * Generic exception class for SerDes. + * + */ + +public class SerDeException extends Exception { + private static final long serialVersionUID = 1L; + + public SerDeException() { + super(); + } + + public SerDeException(String message) { + super(message); + } + + public SerDeException(Throwable cause) { + super(cause); + } + + public SerDeException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/SerDeStats.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/SerDeStats.java new file mode 100644 index 0000000000000000000000000000000000000000..6cf2ccdcffff06bae21986c3df4c29fad8b560c5 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/SerDeStats.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2; + +public class SerDeStats { + + /** + * Class used to pass statistics information from serializer/deserializer to the tasks. + * A SerDeStats object is returned by calling SerDe.getStats(). + */ + + // currently we support only raw data size stat + private long rawDataSize; + private long rowCount; + + public SerDeStats() { + rawDataSize = 0; + rowCount = 0; + } + + /** + * Return the raw data size + * @return raw data size + */ + public long getRawDataSize() { + return rawDataSize; + } + + /** + * Set the raw data size + * @param uSize - size to be set + */ + public void setRawDataSize(long uSize) { + rawDataSize = uSize; + } + + /** + * Return the row count + * @return row count + */ + public long getRowCount() { + return rowCount; + } + + /** + * Set the row count + * @param rowCount - count of rows + */ + public void setRowCount(long rowCount) { + this.rowCount = rowCount; + } + +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/Serializer.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/Serializer.java new file mode 100644 index 0000000000000000000000000000000000000000..3f07a86e31cdcb1ee082cda42163a192362d14ea --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/Serializer.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2; + +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.io.Writable; + +/** + * HiveSerializer is used to serialize data to a Hadoop Writable object. The + * serialize In addition to the interface below, all implementations are assume + * to have a ctor that takes a single 'Table' object as argument. + * All serializers should extend the abstract class AbstractSerializer. + * The interface is necessary for SerDes to be able to implement both Serializer and Deserializer. + */ +public interface Serializer { + + /** + * Initialize the HiveSerializer. + * + * @param conf + * System properties + * @param tbl + * table properties + * @throws SerDeException + */ + void initialize(Configuration conf, Properties tbl) throws SerDeException; + + /** + * Returns the Writable class that would be returned by the serialize method. + * This is used to initialize SequenceFile header. + */ + Class getSerializedClass(); + + /** + * Serialize an object by navigating inside the Object with the + * ObjectInspector. In most cases, the return value of this function will be + * constant since the function will reuse the Writable object. If the client + * wants to keep a copy of the Writable, the client needs to clone the + * returned value. + */ + Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException; + + /** + * Returns statistics collected when serializing + */ + SerDeStats getSerDeStats(); +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/ConstantObjectInspector.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/ConstantObjectInspector.java new file mode 100644 index 0000000000000000000000000000000000000000..2b98e58619f126a40c75b87366110c5f48daabb1 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/ConstantObjectInspector.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * ConstantObjectInspector. This interface should be implemented by + * ObjectInspectors which represent constant values and can return them without + * an evaluation. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public interface ConstantObjectInspector extends ObjectInspector { + + Object getWritableConstantValue(); + +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectInspector.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectInspector.java new file mode 100644 index 0000000000000000000000000000000000000000..f8409398b6b1ac9a5fcad3f488afe31eeb8a80ff --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectInspector.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import java.util.List; + +/** + * ListObjectInspector. + * + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public interface ListObjectInspector extends ObjectInspector { + + // ** Methods that does not need a data object ** + ObjectInspector getListElementObjectInspector(); + + // ** Methods that need a data object ** + /** + * returns null for null list, out-of-the-range index. + */ + Object getListElement(Object data, int index); + + /** + * returns -1 for data = null. + */ + int getListLength(Object data); + + /** + * returns null for data = null. + * + * Note: This method should not return a List object that is reused by the + * same ListObjectInspector, because it's possible that the same + * ListObjectInspector will be used in multiple places in the code. + * + * However it's OK if the List object is part of the Object data. + */ + List getList(Object data); + +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapObjectInspector.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapObjectInspector.java new file mode 100644 index 0000000000000000000000000000000000000000..f0b8a66c5544a91c9a46d24c8ecbf1f1bd8e55b7 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapObjectInspector.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import java.util.Map; + +/** + * MapObjectInspector. + * + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public interface MapObjectInspector extends ObjectInspector { + + // ** Methods that does not need a data object ** + // Map Type + ObjectInspector getMapKeyObjectInspector(); + + ObjectInspector getMapValueObjectInspector(); + + // ** Methods that need a data object ** + // In this function, key has to be of the same structure as the Map expects. + // Most cases key will be primitive type, so it's OK. + // In rare cases that key is not primitive, the user is responsible for + // defining + // the hashCode() and equals() methods of the key class. + Object getMapValueElement(Object data, Object key); + + /** + * returns null for data = null. + * + * Note: This method should not return a Map object that is reused by the same + * MapObjectInspector, because it's possible that the same MapObjectInspector + * will be used in multiple places in the code. + * + * However it's OK if the Map object is part of the Object data. + */ + Map getMap(Object data); + + /** + * returns -1 for NULL map. + */ + int getMapSize(Object data); +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java new file mode 100644 index 0000000000000000000000000000000000000000..8c35debd34aa5bbbe85478c0f0fba581dad762f2 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.objectinspector; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +/** + * ObjectInspector helps us to look into the internal structure of a complex + * object. + * + * A (probably configured) ObjectInspector instance stands for a specific type + * and a specific way to store the data of that type in the memory. + * + * For native java Object, we can directly access the internal structure through + * member fields and methods. ObjectInspector is a way to delegate that + * functionality away from the Object, so that we have more control on the + * behavior of those actions. + * + * An efficient implementation of ObjectInspector should rely on factory, so + * that we can make sure the same ObjectInspector only has one instance. That + * also makes sure hashCode() and equals() methods of java.lang.Object directly + * works for ObjectInspector as well. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public interface ObjectInspector extends Cloneable { + + /** + * Category. + * + */ + public static enum Category { + PRIMITIVE, LIST, MAP, STRUCT, UNION + }; + + /** + * Returns the name of the data type that is inspected by this + * ObjectInspector. This is used to display the type information to the user. + * + * For primitive types, the type name is standardized. For other types, the + * type name can be something like "list<int>", "map<int,string>", java class + * names, or user-defined type names similar to typedef. + */ + String getTypeName(); + + /** + * An ObjectInspector must inherit from one of the following interfaces if + * getCategory() returns: PRIMITIVE: PrimitiveObjectInspector LIST: + * ListObjectInspector MAP: MapObjectInspector STRUCT: StructObjectInspector. + */ + Category getCategory(); +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java new file mode 100644 index 0000000000000000000000000000000000000000..e08fd10213bf6fcf565ec1970e86e3c00db1cf24 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; + + +/** + * PrimitiveObjectInspector. + * + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public interface PrimitiveObjectInspector extends ObjectInspector { + + /** + * The primitive types supported by Hive. + */ + public static enum PrimitiveCategory { + VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, + DATE, TIMESTAMP, TIMESTAMPLOCALTZ, BINARY, DECIMAL, VARCHAR, CHAR, + INTERVAL_YEAR_MONTH, INTERVAL_DAY_TIME, UNKNOWN + }; + + public PrimitiveTypeInfo getTypeInfo(); + + /** + * Get the primitive category of the PrimitiveObjectInspector. + */ + PrimitiveCategory getPrimitiveCategory(); + + /** + * Get the Primitive Writable class which is the return type of + * getPrimitiveWritableObject() and copyToPrimitiveWritableObject(). + */ + Class getPrimitiveWritableClass(); + + /** + * Return the data in an instance of primitive writable Object. If the Object + * is already a primitive writable Object, just return o. + */ + Object getPrimitiveWritableObject(Object o); + + /** + * Get the Java Primitive class which is the return type of + * getJavaPrimitiveObject(). + */ + Class getJavaPrimitiveClass(); + + /** + * Get the Java Primitive object. + */ + Object getPrimitiveJavaObject(Object o); + + /** + * Get a copy of the Object in the same class, so the return value can be + * stored independently of the parameter. + * + * If the Object is a Primitive Java Object, we just return the parameter + * since Primitive Java Object is immutable. + */ + Object copyObject(Object o); + + /** + * Whether the ObjectInspector prefers to return a Primitive Writable Object + * instead of a Primitive Java Object. This can be useful for determining the + * most efficient way to getting data out of the Object. + */ + boolean preferWritable(); + + /** + * The precision of the underlying data. + */ + int precision(); + + /** + * The scale of the underlying data. + */ + int scale(); + +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java new file mode 100644 index 0000000000000000000000000000000000000000..566b0a4690017e69327eb7e2db54de0efada4bd8 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.objectinspector; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Classes implementing this interface are considered to represent a field of a + * struct for this serde package. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public interface StructField { + + /** + * Get the name of the field. The name should be always in lower-case. + */ + String getFieldName(); + + /** + * Get the ObjectInspector for the field. + */ + ObjectInspector getFieldObjectInspector(); + + /** + * Get the fieldID for the field. + */ + int getFieldID(); + + /** + * Get the comment for the field. May be null if no comment provided. + */ + String getFieldComment(); +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructObjectInspector.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructObjectInspector.java new file mode 100644 index 0000000000000000000000000000000000000000..347f4fd6c4a3634ca52bfb3db7b555e06d98a87a --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructObjectInspector.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import java.util.List; + +/** + * StructObjectInspector. + * + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public abstract class StructObjectInspector implements ObjectInspector { + + // ** Methods that does not need a data object ** + /** + * Returns all the fields. + */ + public abstract List getAllStructFieldRefs(); + + /** + * Look up a field. + */ + public abstract StructField getStructFieldRef(String fieldName); + + // ** Methods that need a data object ** + /** + * returns null for data = null. + */ + public abstract Object getStructFieldData(Object data, StructField fieldRef); + + /** + * returns null for data = null. + */ + public abstract List getStructFieldsDataAsList(Object data); + + public boolean isSettable() { + return false; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + List fields = getAllStructFieldRefs(); + sb.append(getClass().getName()); + sb.append("<"); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append(fields.get(i).getFieldObjectInspector().toString()); + } + sb.append(">"); + return sb.toString(); + } +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObjectInspector.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObjectInspector.java new file mode 100644 index 0000000000000000000000000000000000000000..af3604d0a301fd1edbf9272835a6f1446bdbb492 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionObjectInspector.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.objectinspector; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import java.util.List; + +/** + * UnionObjectInspector works on union data that is stored as UnionObject. + * + * It holds the list of the object inspectors corresponding to each type of the + * object the Union can hold. + * + * UnionObjectInspector. + * + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public interface UnionObjectInspector extends ObjectInspector { + + /** + * Returns the array of ObjectInspectors that are for each of the tags. + */ + List getObjectInspectors(); + + /** + * Return the tag of the object. + */ + byte getTag(Object o); + + /** + * Return the field based on the tag associated with the Object. + */ + Object getField(Object o); + +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveTypeEntry.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveTypeEntry.java new file mode 100644 index 0000000000000000000000000000000000000000..b3c8d04dc7d745119feb9d82265e708b4c9d273f --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveTypeEntry.java @@ -0,0 +1,155 @@ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * TypeEntry stores information about a Hive Primitive TypeInfo. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public class PrimitiveTypeEntry implements Writable, Cloneable { + + private static final Map + primitiveCategoryToTypeEntry = + new HashMap<>(); + private static final Map, PrimitiveTypeEntry> primitiveJavaTypeToTypeEntry = + new HashMap<>(); + private static final Map, PrimitiveTypeEntry> primitiveJavaClassToTypeEntry = + new HashMap<>(); + private static final Map, PrimitiveTypeEntry> primitiveWritableClassToTypeEntry = + new HashMap<>(); + + // Base type name to PrimitiveTypeEntry map. + private static final Map typeNameToTypeEntry = + new HashMap<>(); + + public static void addParameterizedType(PrimitiveTypeEntry t) { + typeNameToTypeEntry.put(t.toString(), t); + } + + public static void registerType(PrimitiveTypeEntry t) { + if (t.primitiveCategory != PrimitiveObjectInspector.PrimitiveCategory.UNKNOWN) { + primitiveCategoryToTypeEntry.put(t.primitiveCategory, t); + } + if (t.primitiveJavaType != null) { + primitiveJavaTypeToTypeEntry.put(t.primitiveJavaType, t); + } + if (t.primitiveJavaClass != null) { + primitiveJavaClassToTypeEntry.put(t.primitiveJavaClass, t); + } + if (t.primitiveWritableClass != null) { + primitiveWritableClassToTypeEntry.put(t.primitiveWritableClass, t); + } + if (t.typeName != null) { + typeNameToTypeEntry.put(t.typeName, t); + } + } + + public static PrimitiveTypeEntry fromJavaType(Class clazz) { + return primitiveJavaTypeToTypeEntry.get(clazz); + } + + public static PrimitiveTypeEntry fromJavaClass(Class clazz) { + return primitiveJavaClassToTypeEntry.get(clazz); + } + + public static PrimitiveTypeEntry fromWritableClass(Class clazz) { + return primitiveWritableClassToTypeEntry.get(clazz); + } + + public static PrimitiveTypeEntry fromPrimitiveCategory( + PrimitiveObjectInspector.PrimitiveCategory category) { + return primitiveCategoryToTypeEntry.get(category); + } + + public static PrimitiveTypeEntry fromTypeName(String typeName) { + return typeNameToTypeEntry.get(typeName); + } + /** + * The category of the PrimitiveType. + */ + public PrimitiveObjectInspector.PrimitiveCategory primitiveCategory; + + /** + * primitiveJavaType refers to java types like int, double, etc. + */ + public Class primitiveJavaType; + /** + * primitiveJavaClass refers to java classes like Integer, Double, String + * etc. + */ + public Class primitiveJavaClass; + /** + * writableClass refers to hadoop Writable classes like IntWritable, + * DoubleWritable, Text etc. + */ + public Class primitiveWritableClass; + /** + * typeName is the name of the type as in DDL. + */ + public String typeName; + + protected PrimitiveTypeEntry() { + super(); + } + + public PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory primitiveCategory, + String typeName, Class primitiveType, Class javaClass, Class hiveClass) { + this.primitiveCategory = primitiveCategory; + primitiveJavaType = primitiveType; + primitiveJavaClass = javaClass; + primitiveWritableClass = hiveClass; + this.typeName = typeName; + } + + @Override + public void readFields(DataInput in) throws IOException { + primitiveCategory = WritableUtils.readEnum(in, + PrimitiveObjectInspector.PrimitiveCategory.class); + typeName = WritableUtils.readString(in); + try { + primitiveJavaType = Class.forName(WritableUtils.readString(in)); + primitiveJavaClass = Class.forName(WritableUtils.readString(in)); + primitiveWritableClass = Class.forName(WritableUtils.readString(in)); + } catch (ClassNotFoundException e) { + throw new IOException(e); + } + } + + @Override + public void write(DataOutput out) throws IOException { + + WritableUtils.writeEnum(out, primitiveCategory); + WritableUtils.writeString(out, typeName); + WritableUtils.writeString(out, primitiveJavaType.getName()); + WritableUtils.writeString(out, primitiveJavaClass.getName()); + WritableUtils.writeString(out, primitiveWritableClass.getName()); + } + + @Override + public Object clone() { + PrimitiveTypeEntry result = new PrimitiveTypeEntry( + this.primitiveCategory, + this.typeName, + this.primitiveJavaType, + this.primitiveJavaClass, + this.primitiveWritableClass); + return result; + } + + @Override + public String toString() { + return typeName; + } + +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..2ccaf4d8ebc4fb7ac20b20d163c1c05edc3a6e53 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +@InterfaceAudience.Public +@InterfaceStability.Stable +public abstract class BaseCharTypeInfo extends PrimitiveTypeInfo { + private static final long serialVersionUID = 1L; + + private int length; + + // no-arg constructor to make kyro happy. + public BaseCharTypeInfo() { + } + + public BaseCharTypeInfo(String typeName) { + super(typeName); + } + + public BaseCharTypeInfo(String typeName, int length) { + super(typeName); + this.length = length; + } + + public int getLength() { + return length; + } + + public void setLength(int length) { + this.length = length; + } + + @Override + public String getQualifiedName() { + return getQualifiedName(typeName, length); + } + + public static String getQualifiedName(String typeName, int length) { + StringBuilder sb = new StringBuilder(typeName); + sb.append("("); + sb.append(length); + sb.append(")"); + return sb.toString(); + } + + @Override + public void setTypeName(String typeName) { + // type name should already be set by subclass + return; + } +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..7a455883aca971e6da81c6c658670f141065e26e --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import org.apache.hadoop.hive.serde.serdeConstants; + +public class CharTypeInfo extends BaseCharTypeInfo { + private static final long serialVersionUID = 1L; + + // no-arg constructor to make kyro happy. + public CharTypeInfo() { + super(serdeConstants.CHAR_TYPE_NAME); + } + + public CharTypeInfo(int length) { + super(serdeConstants.CHAR_TYPE_NAME, length); + //TODO : Should the validation be in this TypeInfo? + //BaseCharUtils.validateCharParameter(length); + } + + @Override + public String getTypeName() { + return getQualifiedName(); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + + CharTypeInfo pti = (CharTypeInfo) other; + + return this.typeName.equals(pti.typeName) && this.getLength() == pti.getLength(); + } + + /** + * Generate the hashCode for this TypeInfo. + */ + @Override + public int hashCode() { + return getQualifiedName().hashCode(); + } + + @Override + public String toString() { + return getQualifiedName(); + } + +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..b43a76cb627b282f78967d23912e9351b2935759 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import org.apache.hadoop.hive.serde.serdeConstants; + +public class DecimalTypeInfo extends PrimitiveTypeInfo { + private static final long serialVersionUID = 1L; + + private int precision; + private int scale; + + // no-arg constructor to make kyro happy. + public DecimalTypeInfo() { + super(serdeConstants.DECIMAL_TYPE_NAME); + } + + public DecimalTypeInfo(int precision, int scale) { + super(serdeConstants.DECIMAL_TYPE_NAME); + //TODO : Should the validation be in this TypeInfo? + //HiveDecimalUtils.validateParameter(precision, scale); + this.precision = precision; + this.scale = scale; + } + + @Override + public String getTypeName() { + return getQualifiedName(); + } + + @Override + public void setTypeName(String typeName) { + // No need to set type name, it should always be decimal + return; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + + DecimalTypeInfo dti = (DecimalTypeInfo)other; + + return this.precision() == dti.precision() && this.scale() == dti.scale(); + + } + + /** + * Generate the hashCode for this TypeInfo. + */ + @Override + public int hashCode() { + return 31 * (17 + precision) + scale; + } + + @Override + public String toString() { + return getQualifiedName(); + } + + @Override + public String getQualifiedName() { + return getQualifiedName(precision, scale); + } + + public static String getQualifiedName(int precision, int scale) { + StringBuilder sb = new StringBuilder(serdeConstants.DECIMAL_TYPE_NAME); + sb.append("("); + sb.append(precision); + sb.append(","); + sb.append(scale); + sb.append(")"); + return sb.toString(); + } + + public int precision() { + return precision; + } + + public int scale() { + return scale; + } + + @Override + public boolean accept(TypeInfo other) { + if (other == null || getClass() != other.getClass()) { + return false; + } + + DecimalTypeInfo dti = (DecimalTypeInfo)other; + // Make sure "this" has enough integer room to accomodate other's integer digits. + return this.precision() - this.scale() >= dti.precision() - dti.scale(); + } + + public int getPrecision() { + return precision; + } + + public void setPrecision(int precision) { + this.precision = precision; + } + + public int getScale() { + return scale; + } + + public void setScale(int scale) { + this.scale = scale; + } + +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..c2e21fd36622c70344a5bbc585d3a2f80cc5556c --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serializable; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; + +/** + * A List Type has homogeneous elements. All elements of the List has the same + * TypeInfo which is returned by getListElementTypeInfo. + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public final class ListTypeInfo extends TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + private TypeInfo listElementTypeInfo; + + /** + * For java serialization use only. + */ + public ListTypeInfo() { + } + + @Override + public String getTypeName() { + return org.apache.hadoop.hive.serde.serdeConstants.LIST_TYPE_NAME + "<" + + listElementTypeInfo.getTypeName() + ">"; + } + + /** + * For java serialization use only. + */ + public void setListElementTypeInfo(TypeInfo listElementTypeInfo) { + this.listElementTypeInfo = listElementTypeInfo; + } + + /** + * For TypeInfoFactory use only. + */ + ListTypeInfo(TypeInfo elementTypeInfo) { + listElementTypeInfo = elementTypeInfo; + } + + @Override + public Category getCategory() { + return Category.LIST; + } + + public TypeInfo getListElementTypeInfo() { + return listElementTypeInfo; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof ListTypeInfo)) { + return false; + } + return getListElementTypeInfo().equals( + ((ListTypeInfo) other).getListElementTypeInfo()); + } + + @Override + public int hashCode() { + return listElementTypeInfo.hashCode(); + } + +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..b46a367fe4f29ecd44618064ac7353895d6e9eb2 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serializable; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; + +/** + * A Map Type has homogeneous keys and homogeneous values. All keys of the Map + * have the same TypeInfo, which is returned by getMapKeyTypeInfo(); and all + * values of the Map has the same TypeInfo, which is returned by + * getMapValueTypeInfo(). + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public final class MapTypeInfo extends TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + + private TypeInfo mapKeyTypeInfo; + private TypeInfo mapValueTypeInfo; + + /** + * For java serialization use only. + */ + public MapTypeInfo() { + } + + @Override + public String getTypeName() { + return org.apache.hadoop.hive.serde.serdeConstants.MAP_TYPE_NAME + "<" + + mapKeyTypeInfo.getTypeName() + "," + mapValueTypeInfo.getTypeName() + + ">"; + } + + /** + * For java serialization use only. + */ + public void setMapKeyTypeInfo(TypeInfo mapKeyTypeInfo) { + this.mapKeyTypeInfo = mapKeyTypeInfo; + } + + /** + * For java serialization use only. + */ + public void setMapValueTypeInfo(TypeInfo mapValueTypeInfo) { + this.mapValueTypeInfo = mapValueTypeInfo; + } + + // For TypeInfoFactory use only + MapTypeInfo(TypeInfo keyTypeInfo, TypeInfo valueTypeInfo) { + mapKeyTypeInfo = keyTypeInfo; + mapValueTypeInfo = valueTypeInfo; + } + + @Override + public Category getCategory() { + return Category.MAP; + } + + public TypeInfo getMapKeyTypeInfo() { + return mapKeyTypeInfo; + } + + public TypeInfo getMapValueTypeInfo() { + return mapValueTypeInfo; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof MapTypeInfo)) { + return false; + } + MapTypeInfo o = (MapTypeInfo) other; + return o.getMapKeyTypeInfo().equals(getMapKeyTypeInfo()) + && o.getMapValueTypeInfo().equals(getMapValueTypeInfo()); + } + + @Override + public int hashCode() { + return mapKeyTypeInfo.hashCode() ^ mapValueTypeInfo.hashCode(); + } + +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..cbacbbd889b5a9658840d498705969937a9cc9dc --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveTypeEntry; + +/** + * There are limited number of Primitive Types. All Primitive Types are defined + * by TypeInfoFactory.isPrimitiveClass(). + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public class PrimitiveTypeInfo extends TypeInfo implements Serializable { + private static final long serialVersionUID = 1L; + + // Base name (varchar vs fully qualified name such as varchar(200)). + protected String typeName; + + /** + * For java serialization use only. + */ + public PrimitiveTypeInfo() { + } + + /** + * For TypeInfoFactory use only. + */ + PrimitiveTypeInfo(String typeName) { + this.typeName = typeName; + } + + /** + * Returns the category of this TypeInfo. + */ + @Override + public Category getCategory() { + return Category.PRIMITIVE; + } + + public PrimitiveCategory getPrimitiveCategory() { + return getPrimitiveTypeEntry().primitiveCategory; + } + + public Class getPrimitiveWritableClass() { + return getPrimitiveTypeEntry().primitiveWritableClass; + } + + public Class getPrimitiveJavaClass() { + return getPrimitiveTypeEntry().primitiveJavaClass; + } + + // The following 2 methods are for java serialization use only. + public void setTypeName(String typeName) { + this.typeName = typeName; + } + + @Override + public String getTypeName() { + return typeName; + } + + public PrimitiveTypeEntry getPrimitiveTypeEntry() { + return PrimitiveTypeEntry.fromTypeName(typeName); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + + PrimitiveTypeInfo pti = (PrimitiveTypeInfo) other; + + return this.typeName.equals(pti.typeName); + } + + /** + * Generate the hashCode for this TypeInfo. + */ + @Override + public int hashCode() { + return typeName.hashCode(); + } + + @Override + public String toString() { + return typeName; + } +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfoValidationUtils.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfoValidationUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..164560a8f55b63d33606f900e077f92ebfba3002 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfoValidationUtils.java @@ -0,0 +1,37 @@ +package org.apache.hadoop.hive.serde2.typeinfo; + +import org.apache.hadoop.hive.common.type.HiveChar; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveVarchar; + +public class PrimitiveTypeInfoValidationUtils { + public static void validateVarcharParameter(int length) { + if (length > HiveVarchar.MAX_VARCHAR_LENGTH || length < 1) { + throw new RuntimeException("Varchar length " + length + " out of allowed range [1, " + + HiveVarchar.MAX_VARCHAR_LENGTH + "]"); + } + } + + public static void validateCharParameter(int length) { + if (length > HiveChar.MAX_CHAR_LENGTH || length < 1) { + throw new RuntimeException("Char length " + length + " out of allowed range [1, " + + HiveChar.MAX_CHAR_LENGTH + "]"); + } + } + + public static void validateParameter(int precision, int scale) { + if (precision < 1 || precision > HiveDecimal.MAX_PRECISION) { + throw new IllegalArgumentException("Decimal precision out of allowed range [1," + + HiveDecimal.MAX_PRECISION + "]"); + } + + if (scale < 0 || scale > HiveDecimal.MAX_SCALE) { + throw new IllegalArgumentException("Decimal scale out of allowed range [0," + + HiveDecimal.MAX_SCALE + "]"); + } + + if (precision < scale) { + throw new IllegalArgumentException("Decimal scale must be less than or equal to precision"); + } + } +} \ No newline at end of file diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..202be14e8d91735594a8c64a091a80fa3a568136 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; + +/** + * StructTypeInfo represents the TypeInfo of a struct. A struct contains one or + * more fields each of which has a unique name and its own TypeInfo. Different + * fields can have the same or different TypeInfo. + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public final class StructTypeInfo extends TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + + private ArrayList allStructFieldNames; + private ArrayList allStructFieldTypeInfos; + + /** + * For java serialization use only. + */ + public StructTypeInfo() { + } + + @Override + public String getTypeName() { + StringBuilder sb = new StringBuilder(); + sb.append(serdeConstants.STRUCT_TYPE_NAME + "<"); + for (int i = 0; i < allStructFieldNames.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append(allStructFieldNames.get(i)); + sb.append(":"); + sb.append(allStructFieldTypeInfos.get(i).getTypeName()); + } + sb.append(">"); + return sb.toString(); + } + + /** + * For java serialization use only. + */ + public void setAllStructFieldNames(ArrayList allStructFieldNames) { + this.allStructFieldNames = allStructFieldNames; + } + + /** + * For java serialization use only. + */ + public void setAllStructFieldTypeInfos( + ArrayList allStructFieldTypeInfos) { + this.allStructFieldTypeInfos = allStructFieldTypeInfos; + } + + /** + * For TypeInfoFactory use only. + */ + StructTypeInfo(List names, List typeInfos) { + allStructFieldNames = new ArrayList(names); + allStructFieldTypeInfos = new ArrayList(typeInfos); + } + + @Override + public Category getCategory() { + return Category.STRUCT; + } + + public ArrayList getAllStructFieldNames() { + return allStructFieldNames; + } + + public ArrayList getAllStructFieldTypeInfos() { + return allStructFieldTypeInfos; + } + + public TypeInfo getStructFieldTypeInfo(String field) { + String fieldLowerCase = field.toLowerCase(); + for (int i = 0; i < allStructFieldNames.size(); i++) { + if (fieldLowerCase.equalsIgnoreCase(allStructFieldNames.get(i))) { + return allStructFieldTypeInfos.get(i); + } + } + throw new RuntimeException("cannot find field " + field + + "(lowercase form: " + fieldLowerCase + ") in " + allStructFieldNames); + // return null; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof StructTypeInfo)) { + return false; + } + StructTypeInfo o = (StructTypeInfo) other; + Iterator namesIterator = getAllStructFieldNames().iterator(); + Iterator otherNamesIterator = o.getAllStructFieldNames().iterator(); + + // Compare the field names using ignore-case semantics + while (namesIterator.hasNext() && otherNamesIterator.hasNext()) { + if (!namesIterator.next().equalsIgnoreCase(otherNamesIterator.next())) { + return false; + } + } + + // Different number of field names + if (namesIterator.hasNext() || otherNamesIterator.hasNext()) { + return false; + } + + // Compare the field types + return o.getAllStructFieldTypeInfos().equals(getAllStructFieldTypeInfos()); + } + + @Override + public int hashCode() { + return allStructFieldNames.hashCode() ^ allStructFieldTypeInfos.hashCode(); + } + +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..51e73b71f20a8f2492b74ed15d64461f5a22ece4 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.time.DateTimeException; +import java.time.ZoneId; +import java.util.Objects; + +import org.apache.hadoop.hive.serde.serdeConstants; + +public class TimestampLocalTZTypeInfo extends PrimitiveTypeInfo { + private static final long serialVersionUID = 1L; + + private ZoneId timeZone; + + public TimestampLocalTZTypeInfo() { + super(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME); + } + + public TimestampLocalTZTypeInfo(String timeZoneStr) { + super(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME); + this.timeZone = TimestampLocalTZTypeInfo.parseTimeZone(timeZoneStr); + } + + public static ZoneId parseTimeZone(String timeZoneStr) { + if (timeZoneStr == null || timeZoneStr.trim().isEmpty() || + timeZoneStr.trim().toLowerCase().equals("local")) { + // default + return ZoneId.systemDefault(); + } + try { + return ZoneId.of(timeZoneStr); + } catch (DateTimeException e1) { + // default + throw new RuntimeException("Invalid time zone displacement value"); + } + } + + @Override + public String getTypeName() { + return serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME; + } + + @Override + public void setTypeName(String typeName) { + // No need to set type name, it should always be timestamplocaltz + return; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + + TimestampLocalTZTypeInfo dti = (TimestampLocalTZTypeInfo) other; + + return this.timeZone().equals(dti.timeZone()); + } + + /** + * Generate the hashCode for this TypeInfo. + */ + @Override + public int hashCode() { + return Objects.hash(typeName, timeZone); + } + + @Override + public String toString() { + return getQualifiedName(); + } + + @Override + public String getQualifiedName() { + return getQualifiedName(timeZone); + } + + public static String getQualifiedName(ZoneId timeZone) { + StringBuilder sb = new StringBuilder(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME); + sb.append("('"); + sb.append(timeZone); + sb.append("')"); + return sb.toString(); + } + + public ZoneId timeZone() { + return timeZone; + } + + public ZoneId getTimeZone() { + return timeZone; + } + + public void setTimeZone(ZoneId timeZone) { + this.timeZone = timeZone; + } + +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..cc05968fc4e1bf4716adf92fae7dc571d177bc55 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serializable; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; + +/** + * Stores information about a type. Always use the TypeInfoFactory to create new + * TypeInfo objects. + * + * We support 8 categories of types: + * 1. Primitive objects (String, Number, etc) + * 2. List objects (a list of objects of a single type) + * 3. Map objects (a map from objects of one type to objects of another type) + * 4. Struct objects (a list of fields with names and their own types) + * 5. Union objects + * 6. Decimal objects + * 7. Char objects + * 8. Varchar objects + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public abstract class TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + + protected TypeInfo() { + } + + /** + * The Category of this TypeInfo. Possible values are Primitive, List, Map, + * Struct and Union, which corresponds to the 5 sub-classes of TypeInfo. + */ + public abstract Category getCategory(); + + /** + * A String representation of the TypeInfo. + */ + public abstract String getTypeName(); + + /** + * String representing the qualified type name. + * Qualified types should override this method. + * @return + */ + public String getQualifiedName() { + return getTypeName(); + } + + @Override + public String toString() { + return getTypeName(); + } + + @Override + public abstract boolean equals(Object o); + + @Override + public abstract int hashCode(); + + public boolean accept(TypeInfo other) { + return this.equals(other); + } + +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..dfaadafc514af6544f5ae06b6a77265cbdfeeef2 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java @@ -0,0 +1,284 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.hadoop.hive.common.type.HiveChar; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveTypeEntry; + +/** + * TypeInfoFactory can be used to create the TypeInfo object for any types. + * + * TypeInfo objects are all read-only so we can reuse them easily. + * TypeInfoFactory has internal cache to make sure we don't create 2 TypeInfo + * objects that represents the same type. + */ +public final class TypeInfoFactory { + + private TypeInfoFactory() { + // prevent instantiation + } + + public static final PrimitiveTypeInfo voidTypeInfo = new PrimitiveTypeInfo(serdeConstants.VOID_TYPE_NAME); + public static final PrimitiveTypeInfo booleanTypeInfo = new PrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME); + public static final PrimitiveTypeInfo intTypeInfo = new PrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME); + public static final PrimitiveTypeInfo longTypeInfo = new PrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME); + public static final PrimitiveTypeInfo stringTypeInfo = new PrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); + public static final PrimitiveTypeInfo charTypeInfo = new CharTypeInfo(HiveChar.MAX_CHAR_LENGTH); + public static final PrimitiveTypeInfo varcharTypeInfo = new VarcharTypeInfo(HiveVarchar.MAX_VARCHAR_LENGTH); + public static final PrimitiveTypeInfo floatTypeInfo = new PrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME); + public static final PrimitiveTypeInfo doubleTypeInfo = new PrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME); + public static final PrimitiveTypeInfo byteTypeInfo = new PrimitiveTypeInfo(serdeConstants.TINYINT_TYPE_NAME); + public static final PrimitiveTypeInfo shortTypeInfo = new PrimitiveTypeInfo(serdeConstants.SMALLINT_TYPE_NAME); + public static final PrimitiveTypeInfo dateTypeInfo = new PrimitiveTypeInfo(serdeConstants.DATE_TYPE_NAME); + public static final PrimitiveTypeInfo timestampTypeInfo = new PrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME); + public static final PrimitiveTypeInfo intervalYearMonthTypeInfo = new PrimitiveTypeInfo(serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME); + public static final PrimitiveTypeInfo intervalDayTimeTypeInfo = new PrimitiveTypeInfo(serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME); + public static final PrimitiveTypeInfo binaryTypeInfo = new PrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME); + + /** + * A DecimalTypeInfo instance that has max precision and max scale. + */ + public static final DecimalTypeInfo decimalTypeInfo = new DecimalTypeInfo(HiveDecimal.SYSTEM_DEFAULT_PRECISION, + HiveDecimal.SYSTEM_DEFAULT_SCALE); + + /** + * A TimestampTZTypeInfo with system default time zone. + */ + public static final TimestampLocalTZTypeInfo timestampLocalTZTypeInfo = new TimestampLocalTZTypeInfo( + ZoneId.systemDefault().getId()); + + public static final PrimitiveTypeInfo unknownTypeInfo = new PrimitiveTypeInfo("unknown"); + + // Map from type name (such as int or varchar(40) to the corresponding PrimitiveTypeInfo + // instance. + private static ConcurrentHashMap cachedPrimitiveTypeInfo = + new ConcurrentHashMap(); + static { + cachedPrimitiveTypeInfo.put(serdeConstants.VOID_TYPE_NAME, voidTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.BOOLEAN_TYPE_NAME, booleanTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.INT_TYPE_NAME, intTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.BIGINT_TYPE_NAME, longTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.STRING_TYPE_NAME, stringTypeInfo); + cachedPrimitiveTypeInfo.put(charTypeInfo.getQualifiedName(), charTypeInfo); + cachedPrimitiveTypeInfo.put(varcharTypeInfo.getQualifiedName(), varcharTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.FLOAT_TYPE_NAME, floatTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.DOUBLE_TYPE_NAME, doubleTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.TINYINT_TYPE_NAME, byteTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.SMALLINT_TYPE_NAME, shortTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.DATE_TYPE_NAME, dateTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.TIMESTAMP_TYPE_NAME, timestampTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME, timestampLocalTZTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME, intervalYearMonthTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME, intervalDayTimeTypeInfo); + cachedPrimitiveTypeInfo.put(serdeConstants.BINARY_TYPE_NAME, binaryTypeInfo); + cachedPrimitiveTypeInfo.put(decimalTypeInfo.getQualifiedName(), decimalTypeInfo); + cachedPrimitiveTypeInfo.put("unknown", unknownTypeInfo); + } + + /** + * Get PrimitiveTypeInfo instance for the given type name of a type + * including types with parameters, such as varchar(20). + * + * @param typeName type name possibly with parameters. + * @return aPrimitiveTypeInfo instance + */ + public static PrimitiveTypeInfo getPrimitiveTypeInfo(String typeName) { + PrimitiveTypeInfo result = cachedPrimitiveTypeInfo.get(typeName); + if (result != null) { + return result; + } + + // Not found in the cache. Must be parameterized types. Create it. + result = createPrimitiveTypeInfo(typeName); + if (result == null) { + throw new RuntimeException("Error creating PrimitiveTypeInfo instance for " + typeName); + } + + PrimitiveTypeInfo prev = cachedPrimitiveTypeInfo.putIfAbsent(typeName, result); + if (prev != null) { + result = prev; + } + return result; + } + + /** + * Create PrimitiveTypeInfo instance for the given full name of the type. The returned + * type is one of the parameterized type info such as VarcharTypeInfo. + * + * @param fullName Fully qualified name of the type + * @return PrimitiveTypeInfo instance + */ + private static PrimitiveTypeInfo createPrimitiveTypeInfo(String fullName) { + //TODO logic copied from TypeInfoUtils.getBaseName(fullName). Should we move this TypeInfoUtils as well? + int idx = fullName.indexOf('('); + String baseName = idx == -1 ? fullName : fullName.substring(0, idx); + PrimitiveTypeEntry typeEntry = + PrimitiveTypeEntry.fromTypeName(baseName); + if (null == typeEntry) { + throw new RuntimeException("Unknown type " + fullName); + } + + TypeInfoParser.PrimitiveParts parts = TypeInfoParser.parsePrimitiveParts(fullName); + if (parts.typeParams == null || parts.typeParams.length < 1) { + return null; + } + + switch (typeEntry.primitiveCategory) { + case CHAR: + if (parts.typeParams.length != 1) { + return null; + } + return new CharTypeInfo(Integer.valueOf(parts.typeParams[0])); + case VARCHAR: + if (parts.typeParams.length != 1) { + return null; + } + return new VarcharTypeInfo(Integer.valueOf(parts.typeParams[0])); + case DECIMAL: + if (parts.typeParams.length != 2) { + return null; + } + return new DecimalTypeInfo(Integer.valueOf(parts.typeParams[0]), + Integer.valueOf(parts.typeParams[1])); + case TIMESTAMPLOCALTZ: + if (parts.typeParams.length != 1) { + return null; + } + return new TimestampLocalTZTypeInfo(parts.typeParams[0]); + default: + return null; + } + } + + public static CharTypeInfo getCharTypeInfo(int length) { + String fullName = BaseCharTypeInfo.getQualifiedName(serdeConstants.CHAR_TYPE_NAME, length); + return (CharTypeInfo) getPrimitiveTypeInfo(fullName); + } + + public static VarcharTypeInfo getVarcharTypeInfo(int length) { + String fullName = BaseCharTypeInfo.getQualifiedName(serdeConstants.VARCHAR_TYPE_NAME, length); + return (VarcharTypeInfo) getPrimitiveTypeInfo(fullName); + } + + public static DecimalTypeInfo getDecimalTypeInfo(int precision, int scale) { + String fullName = DecimalTypeInfo.getQualifiedName(precision, scale); + return (DecimalTypeInfo) getPrimitiveTypeInfo(fullName); + }; + + public static TimestampLocalTZTypeInfo getTimestampTZTypeInfo(ZoneId defaultTimeZone) { + String fullName = TimestampLocalTZTypeInfo.getQualifiedName(defaultTimeZone); + return (TimestampLocalTZTypeInfo) getPrimitiveTypeInfo(fullName); + }; + + public static TypeInfo getPrimitiveTypeInfoFromPrimitiveWritable( + Class clazz) { + PrimitiveTypeEntry typeEntry = PrimitiveTypeEntry.fromWritableClass(clazz); + String typeName = typeEntry == null ? null : typeEntry.typeName; + if (typeName == null) { + throw new RuntimeException("Internal error: Cannot get typeName for " + + clazz); + } + return getPrimitiveTypeInfo(typeName); + } + + public static TypeInfo getPrimitiveTypeInfoFromJavaPrimitive(Class clazz) { + PrimitiveTypeEntry t = PrimitiveTypeEntry.fromJavaType(clazz); + if (t == null) { + t = PrimitiveTypeEntry.fromJavaClass(clazz); + } + String typeName = t == null ? null : t.typeName; + return getPrimitiveTypeInfo(typeName); + } + + static ConcurrentHashMap>, TypeInfo> cachedStructTypeInfo = + new ConcurrentHashMap>, TypeInfo>(); + + public static TypeInfo getStructTypeInfo(List names, + List typeInfos) { + ArrayList> signature = new ArrayList>(2); + signature.add(names); + signature.add(typeInfos); + TypeInfo result = cachedStructTypeInfo.get(signature); + if (result == null) { + result = new StructTypeInfo(names, typeInfos); + TypeInfo prev = cachedStructTypeInfo.putIfAbsent(signature, result); + if (prev != null) { + result = prev; + } + } + return result; + } + + static ConcurrentHashMap, TypeInfo> cachedUnionTypeInfo = + new ConcurrentHashMap, TypeInfo>(); + + public static TypeInfo getUnionTypeInfo(List typeInfos) { + TypeInfo result = cachedUnionTypeInfo.get(typeInfos); + if (result == null) { + result = new UnionTypeInfo(typeInfos); + TypeInfo prev = cachedUnionTypeInfo.putIfAbsent(typeInfos, result); + if (prev != null) { + result = prev; + } + } + return result; + } + + static ConcurrentHashMap cachedListTypeInfo = new ConcurrentHashMap(); + + public static TypeInfo getListTypeInfo(TypeInfo elementTypeInfo) { + TypeInfo result = cachedListTypeInfo.get(elementTypeInfo); + if (result == null) { + result = new ListTypeInfo(elementTypeInfo); + TypeInfo prev = cachedListTypeInfo.putIfAbsent(elementTypeInfo, result); + if (prev != null) { + result = prev; + } + } + return result; + } + + static ConcurrentHashMap, TypeInfo> cachedMapTypeInfo = + new ConcurrentHashMap, TypeInfo>(); + + public static TypeInfo getMapTypeInfo(TypeInfo keyTypeInfo, + TypeInfo valueTypeInfo) { + ArrayList signature = new ArrayList(2); + signature.add(keyTypeInfo); + signature.add(valueTypeInfo); + TypeInfo result = cachedMapTypeInfo.get(signature); + if (result == null) { + result = new MapTypeInfo(keyTypeInfo, valueTypeInfo); + TypeInfo prev = cachedMapTypeInfo.putIfAbsent(signature, result); + if (prev != null) { + result = prev; + } + } + return result; + } + +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoParser.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoParser.java new file mode 100644 index 0000000000000000000000000000000000000000..e6a6fa044cdd8678dc0755101aee65a2b0a88c21 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoParser.java @@ -0,0 +1,339 @@ +package org.apache.hadoop.hive.serde2.typeinfo; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveTypeEntry; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +/** + * Parse a recursive TypeInfo list String. For example, the following inputs + * are valid inputs: + * "int,string,map,list>>,list>" + * The separators between TypeInfos can be ",", ":", or ";". + * + * In order to use this class: TypeInfoParser parser = new + * TypeInfoParser("int,string"); ArrayList typeInfos = + * parser.parseTypeInfos(); + */ +@InterfaceStability.Evolving +@InterfaceAudience.LimitedPrivate({"Standalone Metastore", "Hive"}) +public class TypeInfoParser { + + /** + * Make some of the TypeInfo parsing available as a utility. + */ + public static PrimitiveParts parsePrimitiveParts(String typeInfoString) { + TypeInfoParser parser = new TypeInfoParser(typeInfoString); + return parser.parsePrimitiveParts(); + } + + private static class Token { + public int position; + public String text; + public boolean isType; + + @Override + public String toString() { + return "" + position + ":" + text; + } + }; + + private static boolean isTypeChar(char c) { + return Character.isLetterOrDigit(c) || c == '_' || c == '.' || c == ' ' || c == '$'; + } + + /** + * Tokenize the typeInfoString. The rule is simple: all consecutive + * alphadigits and '_', '.' are in one token, and all other characters are + * one character per token. + * + * tokenize("map") should return + * ["map","<","int",",","string",">"] + * + * Note that we add '$' in new Calcite return path. As '$' will not appear + * in any type in Hive, it is safe to do so. + */ + private static ArrayList tokenize(String typeInfoString) { + ArrayList tokens = new ArrayList(0); + int begin = 0; + int end = 1; + while (end <= typeInfoString.length()) { + // last character ends a token? + // if there are quotes, all the text between the quotes + // is considered a single token (this can happen for + // timestamp with local time-zone) + if (begin > 0 && + typeInfoString.charAt(begin - 1) == '(' && + typeInfoString.charAt(begin) == '\'') { + // Ignore starting quote + begin++; + do { + end++; + } while (typeInfoString.charAt(end) != '\''); + } else if (typeInfoString.charAt(begin) == '\'' && + typeInfoString.charAt(begin + 1) == ')') { + // Ignore closing quote + begin++; + end++; + } + if (end == typeInfoString.length() + || !isTypeChar(typeInfoString.charAt(end - 1)) + || !isTypeChar(typeInfoString.charAt(end))) { + Token t = new Token(); + t.position = begin; + t.text = typeInfoString.substring(begin, end); + t.isType = isTypeChar(typeInfoString.charAt(begin)); + tokens.add(t); + begin = end; + } + end++; + } + return tokens; + } + + public TypeInfoParser(String typeInfoString) { + this.typeInfoString = typeInfoString; + typeInfoTokens = tokenize(typeInfoString); + } + + private final String typeInfoString; + private final ArrayList typeInfoTokens; + private ArrayList typeInfos; + private int iToken; + + public ArrayList parseTypeInfos() { + typeInfos = new ArrayList(); + iToken = 0; + while (iToken < typeInfoTokens.size()) { + typeInfos.add(parseType()); + if (iToken < typeInfoTokens.size()) { + Token separator = typeInfoTokens.get(iToken); + if (",".equals(separator.text) || ";".equals(separator.text) + || ":".equals(separator.text)) { + iToken++; + } else { + throw new IllegalArgumentException( + "Error: ',', ':', or ';' expected at position " + + separator.position + " from '" + typeInfoString + "' " + + typeInfoTokens); + } + } + } + return typeInfos; + } + + private Token peek() { + if (iToken < typeInfoTokens.size()) { + return typeInfoTokens.get(iToken); + } else { + return null; + } + } + + private Token expect(String item) { + return expect(item, null); + } + + private Token expect(String item, String alternative) { + if (iToken >= typeInfoTokens.size()) { + throw new IllegalArgumentException("Error: " + item + + " expected at the end of '" + typeInfoString + "'"); + } + Token t = typeInfoTokens.get(iToken); + if (item.equals("type")) { + if (!serdeConstants.LIST_TYPE_NAME.equals(t.text) + && !serdeConstants.MAP_TYPE_NAME.equals(t.text) + && !serdeConstants.STRUCT_TYPE_NAME.equals(t.text) + && !serdeConstants.UNION_TYPE_NAME.equals(t.text) + && null == PrimitiveTypeEntry.fromTypeName(t.text) + && !t.text.equals(alternative)) { + throw new IllegalArgumentException("Error: " + item + + " expected at the position " + t.position + " of '" + + typeInfoString + "' but '" + t.text + "' is found."); + } + } else if (item.equals("name")) { + if (!t.isType && !t.text.equals(alternative)) { + throw new IllegalArgumentException("Error: " + item + + " expected at the position " + t.position + " of '" + + typeInfoString + "' but '" + t.text + "' is found."); + } + } else { + if (!item.equals(t.text) && !t.text.equals(alternative)) { + throw new IllegalArgumentException("Error: " + item + + " expected at the position " + t.position + " of '" + + typeInfoString + "' but '" + t.text + "' is found."); + } + } + iToken++; + return t; + } + + private String[] parseParams() { + List params = new LinkedList(); + + Token t = peek(); + if (t != null && t.text.equals("(")) { + expect("("); + + // checking for null in the for-loop condition prevents null-ptr exception + // and allows us to fail more gracefully with a parsing error. + for(t = peek(); (t == null) || !t.text.equals(")"); t = expect(",",")")) { + params.add(expect("name").text); + } + if (params.size() == 0) { + throw new IllegalArgumentException( + "type parameters expected for type string " + typeInfoString); + } + } + + return params.toArray(new String[params.size()]); + } + + private TypeInfo parseType() { + + Token t = expect("type"); + + // Is this a primitive type? + PrimitiveTypeEntry typeEntry = + PrimitiveTypeEntry.fromTypeName(t.text); + if (typeEntry != null && typeEntry.primitiveCategory != PrimitiveObjectInspector.PrimitiveCategory.UNKNOWN ) { + String[] params = parseParams(); + switch (typeEntry.primitiveCategory) { + case CHAR: + case VARCHAR: + if (params == null || params.length == 0) { + throw new IllegalArgumentException(typeEntry.typeName + + " type is specified without length: " + typeInfoString); + } + + int length = 1; + if (params.length == 1) { + length = Integer.parseInt(params[0]); + if (typeEntry.primitiveCategory == PrimitiveObjectInspector.PrimitiveCategory.VARCHAR) { + PrimitiveTypeInfoValidationUtils.validateVarcharParameter(length); + return TypeInfoFactory.getVarcharTypeInfo(length); + } else { + PrimitiveTypeInfoValidationUtils.validateCharParameter(length); + return TypeInfoFactory.getCharTypeInfo(length); + } + } else if (params.length > 1) { + throw new IllegalArgumentException( + "Type " + typeEntry.typeName+ " only takes one parameter, but " + + params.length + " is seen"); + } + + case DECIMAL: + int precision = HiveDecimal.USER_DEFAULT_PRECISION; + int scale = HiveDecimal.USER_DEFAULT_SCALE; + if (params == null || params.length == 0) { + // It's possible that old metadata still refers to "decimal" as a column type w/o + // precision/scale. In this case, the default (10,0) is assumed. Thus, do nothing here. + } else if (params.length == 1) { + // only precision is specified + precision = Integer.valueOf(params[0]); + PrimitiveTypeInfoValidationUtils.validateParameter(precision, scale); + } else if (params.length == 2) { + // New metadata always have two parameters. + precision = Integer.parseInt(params[0]); + scale = Integer.parseInt(params[1]); + PrimitiveTypeInfoValidationUtils.validateParameter(precision, scale); + } else if (params.length > 2) { + throw new IllegalArgumentException("Type decimal only takes two parameter, but " + + params.length + " is seen"); + } + return TypeInfoFactory.getDecimalTypeInfo(precision, scale); + + default: + return TypeInfoFactory.getPrimitiveTypeInfo(typeEntry.typeName); + } + } + + // Is this a list type? + if (serdeConstants.LIST_TYPE_NAME.equals(t.text)) { + expect("<"); + TypeInfo listElementType = parseType(); + expect(">"); + return TypeInfoFactory.getListTypeInfo(listElementType); + } + + // Is this a map type? + if (serdeConstants.MAP_TYPE_NAME.equals(t.text)) { + expect("<"); + TypeInfo mapKeyType = parseType(); + expect(","); + TypeInfo mapValueType = parseType(); + expect(">"); + return TypeInfoFactory.getMapTypeInfo(mapKeyType, mapValueType); + } + + // Is this a struct type? + if (serdeConstants.STRUCT_TYPE_NAME.equals(t.text)) { + ArrayList fieldNames = new ArrayList(); + ArrayList fieldTypeInfos = new ArrayList(); + boolean first = true; + do { + if (first) { + expect("<"); + first = false; + } else { + Token separator = expect(">", ","); + if (separator.text.equals(">")) { + // end of struct + break; + } + } + Token name = expect("name",">"); + if (name.text.equals(">")) { + break; + } + fieldNames.add(name.text); + expect(":"); + fieldTypeInfos.add(parseType()); + } while (true); + + return TypeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos); + } + // Is this a union type? + if (serdeConstants.UNION_TYPE_NAME.equals(t.text)) { + List objectTypeInfos = new ArrayList(); + boolean first = true; + do { + if (first) { + expect("<"); + first = false; + } else { + Token separator = expect(">", ","); + if (separator.text.equals(">")) { + // end of union + break; + } + } + objectTypeInfos.add(parseType()); + } while (true); + + return TypeInfoFactory.getUnionTypeInfo(objectTypeInfos); + } + + throw new RuntimeException("Internal error parsing position " + + t.position + " of '" + typeInfoString + "'"); + } + + public PrimitiveParts parsePrimitiveParts() { + PrimitiveParts parts = new PrimitiveParts(); + Token t = expect("type"); + parts.typeName = t.text; + parts.typeParams = parseParams(); + return parts; + } + + public static class PrimitiveParts { + public String typeName; + public String[] typeParams; + } +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnionTypeInfo.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnionTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..51797823e0cba78710b6cd4b1955613bac47bf77 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnionTypeInfo.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; + +/** + * UnionTypeInfo represents the TypeInfo of an union. A union holds only one + * field of the specified fields at any point of time. The fields, a Union can + * hold, can have the same or different TypeInfo. + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public class UnionTypeInfo extends TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + + private List allUnionObjectTypeInfos; + + /** + * For java serialization use only. + */ + public UnionTypeInfo() { + } + + @Override + public String getTypeName() { + StringBuilder sb = new StringBuilder(); + sb.append(serdeConstants.UNION_TYPE_NAME + "<"); + for (int i = 0; i < allUnionObjectTypeInfos.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append(allUnionObjectTypeInfos.get(i).getTypeName()); + } + sb.append(">"); + return sb.toString(); + } + + /** + * For java serialization use only. + */ + public void setAllUnionObjectTypeInfos( + List allUnionObjectTypeInfos) { + this.allUnionObjectTypeInfos = allUnionObjectTypeInfos; + } + + /** + * For TypeInfoFactory use only. + */ + UnionTypeInfo(List typeInfos) { + allUnionObjectTypeInfos = new ArrayList(); + allUnionObjectTypeInfos.addAll(typeInfos); + } + + @Override + public Category getCategory() { + return Category.UNION; + } + + public List getAllUnionObjectTypeInfos() { + return allUnionObjectTypeInfos; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof UnionTypeInfo)) { + return false; + } + UnionTypeInfo o = (UnionTypeInfo) other; + + // Compare the field types + return o.getAllUnionObjectTypeInfos().equals(getAllUnionObjectTypeInfos()); + } + + @Override + public int hashCode() { + return allUnionObjectTypeInfos.hashCode(); + } +} diff --git a/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..c12eb2d9b64d2dce5fedc96d41b5484000ad6e95 --- /dev/null +++ b/storage-api/serde-api/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import org.apache.hadoop.hive.serde.serdeConstants; + +public class VarcharTypeInfo extends BaseCharTypeInfo { + private static final long serialVersionUID = 1L; + + // no-arg constructor to make kyro happy. + public VarcharTypeInfo() { + super(serdeConstants.VARCHAR_TYPE_NAME); + } + + public VarcharTypeInfo(int length) { + super(serdeConstants.VARCHAR_TYPE_NAME, length); + //TODO : Should the validation be in this TypeInfo? + //BaseCharUtils.validateVarcharParameter(length); + } + + @Override + public String getTypeName() { + return getQualifiedName(); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + + VarcharTypeInfo pti = (VarcharTypeInfo) other; + + return this.getLength() == pti.getLength(); + } + + /** + * Generate the hashCode for this TypeInfo. + */ + @Override + public int hashCode() { + return getLength(); + } + + @Override + public String toString() { + return getQualifiedName(); + } + +} diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java b/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java deleted file mode 100644 index e5025bf1407aaf51910849d70e4ccda08a70964d..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.hive.common.io.DiskRange; - -/** - * Disk range information class containing disk ranges and total length. - */ -public class DiskRangeInfo { - List diskRanges; // TODO: use DiskRangeList instead - long totalLength; - - public DiskRangeInfo(int indexBaseOffset) { - this.diskRanges = new ArrayList<>(); - // Some data is missing from the stream for PPD uncompressed read (because index offset is - // relative to the entire stream and we only read part of stream if RGs are filtered; unlike - // with compressed data where PPD only filters CBs, so we always get full CB, and index offset - // is relative to CB). To take care of the case when UncompressedStream goes seeking around by - // its incorrect (relative to partial stream) index offset, we will increase the length by our - // offset-relative-to-the-stream, and also account for it in buffers (see createDiskRangeInfo). - // So, index offset now works; as long as noone seeks into this data before the RG (why would - // they), everything works. This is hacky... Stream shouldn't depend on having all the data. - this.totalLength = indexBaseOffset; - } - - public void addDiskRange(DiskRange diskRange) { - diskRanges.add(diskRange); - totalLength += diskRange.getLength(); - } - - public List getDiskRanges() { - return diskRanges; - } - - public long getTotalLength() { - return totalLength; - } -} - diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java b/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java deleted file mode 100644 index 5ab4ee7802fc385ff04e0f4fcd54398ed31073c5..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common; - -/** Simple object pool to prevent GC on small objects passed between threads. */ -public interface Pool { - /** Object helper for objects stored in the pool. */ - public interface PoolObjectHelper { - /** Called to create an object when one cannot be provided. - * @return a newly allocated object - */ - T create(); - /** Called before the object is put in the pool (regardless of whether put succeeds). - * @param t the object to reset - */ - void resetBeforeOffer(T t); - } - - T take(); - void offer(T t); - int size(); -} \ No newline at end of file diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java b/storage-api/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java deleted file mode 100644 index 94b8c58d28219c13f045f6d9e8dd093a4b3607d9..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common; - -import java.util.Arrays; -import java.util.BitSet; - -/** - * An implementation of {@link org.apache.hadoop.hive.common.ValidTxnList} for use by the compactor. - * - * Compaction should only include txns up to smallest open txn (exclussive). - * There may be aborted txns in the snapshot represented by this ValidCompactorTxnList. - * Thus {@link #isTxnRangeValid(long, long)} returns NONE for any range that inluces any unresolved - * transactions. Any txn above {@code highWatermark} is unresolved. - * These produce the logic we need to assure that the compactor only sees records less than the lowest - * open transaction when choosing which files to compact, but that it still ignores aborted - * records when compacting. - * - * See org.apache.hadoop.hive.metastore.txn.TxnUtils#createValidCompactTxnList() for proper - * way to construct this. - */ -public class ValidCompactorTxnList extends ValidReadTxnList { - public ValidCompactorTxnList() { - super(); - } - public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark) { - this(abortedTxnList, abortedBits, highWatermark, Long.MAX_VALUE); - } - /** - * @param abortedTxnList list of all aborted transactions - * @param abortedBits bitset marking whether the corresponding transaction is aborted - * @param highWatermark highest committed transaction to be considered for compaction, - * equivalently (lowest_open_txn - 1). - */ - public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark, long minOpenTxnId) { - // abortedBits should be all true as everything in exceptions are aborted txns - super(abortedTxnList, abortedBits, highWatermark, minOpenTxnId); - if(this.exceptions.length <= 0) { - return; - } - //now that exceptions (aka abortedTxnList) is sorted - int idx = Arrays.binarySearch(this.exceptions, highWatermark); - int lastElementPos; - if(idx < 0) { - int insertionPoint = -idx - 1 ;//see Arrays.binarySearch() JavaDoc - lastElementPos = insertionPoint - 1; - } - else { - lastElementPos = idx; - } - /* - * ensure that we throw out any exceptions above highWatermark to make - * {@link #isTxnValid(long)} faster - */ - this.exceptions = Arrays.copyOf(this.exceptions, lastElementPos + 1); - } - public ValidCompactorTxnList(String value) { - super(value); - } - /** - * Returns org.apache.hadoop.hive.common.ValidTxnList.RangeResponse.ALL if all txns in - * the range are resolved and RangeResponse.NONE otherwise - */ - @Override - public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId) { - return highWatermark >= maxTxnId ? RangeResponse.ALL : RangeResponse.NONE; - } - - @Override - public boolean isTxnAborted(long txnid) { - return Arrays.binarySearch(exceptions, txnid) >= 0; - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java b/storage-api/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java deleted file mode 100644 index ccdd4b727a59b106da0def2d0173c917e1c5c0f7..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common; - -import java.util.Arrays; -import java.util.BitSet; - -/** - * An implementation of {@link org.apache.hadoop.hive.common.ValidTxnList} for use by readers. - * This class will view a transaction as valid only if it is committed. Both open and aborted - * transactions will be seen as invalid. - */ -public class ValidReadTxnList implements ValidTxnList { - - protected long[] exceptions; - protected BitSet abortedBits; // BitSet for flagging aborted transactions. Bit is true if aborted, false if open - //default value means there are no open txn in the snapshot - private long minOpenTxn = Long.MAX_VALUE; - protected long highWatermark; - - public ValidReadTxnList() { - this(new long[0], new BitSet(), Long.MAX_VALUE, Long.MAX_VALUE); - } - - /** - * Used if there are no open transactions in the snapshot - */ - public ValidReadTxnList(long[] exceptions, BitSet abortedBits, long highWatermark) { - this(exceptions, abortedBits, highWatermark, Long.MAX_VALUE); - } - public ValidReadTxnList(long[] exceptions, BitSet abortedBits, long highWatermark, long minOpenTxn) { - if (exceptions.length > 0) { - this.minOpenTxn = minOpenTxn; - } - this.exceptions = exceptions; - this.abortedBits = abortedBits; - this.highWatermark = highWatermark; - } - - public ValidReadTxnList(String value) { - readFromString(value); - } - - @Override - public boolean isTxnValid(long txnid) { - if (highWatermark < txnid) { - return false; - } - return Arrays.binarySearch(exceptions, txnid) < 0; - } - - /** - * We cannot use a base file if its range contains an open txn. - * @param txnid from base_xxxx - */ - @Override - public boolean isValidBase(long txnid) { - return minOpenTxn > txnid && txnid <= highWatermark; - } - @Override - public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId) { - // check the easy cases first - if (highWatermark < minTxnId) { - return RangeResponse.NONE; - } else if (exceptions.length > 0 && exceptions[0] > maxTxnId) { - return RangeResponse.ALL; - } - - // since the exceptions and the range in question overlap, count the - // exceptions in the range - long count = Math.max(0, maxTxnId - highWatermark); - for(long txn: exceptions) { - if (minTxnId <= txn && txn <= maxTxnId) { - count += 1; - } - } - - if (count == 0) { - return RangeResponse.ALL; - } else if (count == (maxTxnId - minTxnId + 1)) { - return RangeResponse.NONE; - } else { - return RangeResponse.SOME; - } - } - - @Override - public String toString() { - return writeToString(); - } - - @Override - public String writeToString() { - StringBuilder buf = new StringBuilder(); - buf.append(highWatermark); - buf.append(':'); - buf.append(minOpenTxn); - if (exceptions.length == 0) { - buf.append(':'); // separator for open txns - buf.append(':'); // separator for aborted txns - } else { - StringBuilder open = new StringBuilder(); - StringBuilder abort = new StringBuilder(); - for (int i = 0; i < exceptions.length; i++) { - if (abortedBits.get(i)) { - if (abort.length() > 0) { - abort.append(','); - } - abort.append(exceptions[i]); - } else { - if (open.length() > 0) { - open.append(','); - } - open.append(exceptions[i]); - } - } - buf.append(':'); - buf.append(open); - buf.append(':'); - buf.append(abort); - } - return buf.toString(); - } - - @Override - public void readFromString(String src) { - if (src == null || src.length() == 0) { - highWatermark = Long.MAX_VALUE; - exceptions = new long[0]; - abortedBits = new BitSet(); - } else { - String[] values = src.split(":"); - highWatermark = Long.parseLong(values[0]); - minOpenTxn = Long.parseLong(values[1]); - String[] openTxns = new String[0]; - String[] abortedTxns = new String[0]; - if (values.length < 3) { - openTxns = new String[0]; - abortedTxns = new String[0]; - } else if (values.length == 3) { - if (!values[2].isEmpty()) { - openTxns = values[2].split(","); - } - } else { - if (!values[2].isEmpty()) { - openTxns = values[2].split(","); - } - if (!values[3].isEmpty()) { - abortedTxns = values[3].split(","); - } - } - exceptions = new long[openTxns.length + abortedTxns.length]; - int i = 0; - for (String open : openTxns) { - exceptions[i++] = Long.parseLong(open); - } - for (String abort : abortedTxns) { - exceptions[i++] = Long.parseLong(abort); - } - Arrays.sort(exceptions); - abortedBits = new BitSet(exceptions.length); - for (String abort : abortedTxns) { - int index = Arrays.binarySearch(exceptions, Long.parseLong(abort)); - abortedBits.set(index); - } - } - } - - @Override - public long getHighWatermark() { - return highWatermark; - } - - @Override - public long[] getInvalidTransactions() { - return exceptions; - } - - @Override - public Long getMinOpenTxn() { - return minOpenTxn == Long.MAX_VALUE ? null : minOpenTxn; - } - - @Override - public boolean isTxnAborted(long txnid) { - int index = Arrays.binarySearch(exceptions, txnid); - return index >= 0 && abortedBits.get(index); - } - - @Override - public RangeResponse isTxnRangeAborted(long minTxnId, long maxTxnId) { - // check the easy cases first - if (highWatermark < minTxnId) { - return RangeResponse.NONE; - } - - int count = 0; // number of aborted txns found in exceptions - - // traverse the aborted txns list, starting at first aborted txn index - for (int i = abortedBits.nextSetBit(0); i >= 0; i = abortedBits.nextSetBit(i + 1)) { - long abortedTxnId = exceptions[i]; - if (abortedTxnId > maxTxnId) { // we've already gone beyond the specified range - break; - } - if (abortedTxnId >= minTxnId && abortedTxnId <= maxTxnId) { - count++; - } - } - - if (count == 0) { - return RangeResponse.NONE; - } else if (count == (maxTxnId - minTxnId + 1)) { - return RangeResponse.ALL; - } else { - return RangeResponse.SOME; - } - } -} - diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java b/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java deleted file mode 100644 index 108e5ca85cfab26403e74adefd7f349d4105ee6b..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java +++ /dev/null @@ -1,112 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common; - -/** - * Models the list of transactions that should be included in a snapshot. - * It is modelled as a high water mark, which is the largest transaction id that - * has been committed and a list of transactions that are not included. - */ -public interface ValidTxnList { - - /** - * Key used to store valid txn list in a - * {@link org.apache.hadoop.conf.Configuration} object. - */ - public static final String VALID_TXNS_KEY = "hive.txn.valid.txns"; - - /** - * The response to a range query. NONE means no values in this range match, - * SOME mean that some do, and ALL means that every value does. - */ - public enum RangeResponse {NONE, SOME, ALL}; - - /** - * Indicates whether a given transaction is valid. Note that valid may have different meanings - * for different implementations, as some will only want to see committed transactions and some - * both committed and aborted. - * @param txnid id for the transaction - * @return true if valid, false otherwise - */ - public boolean isTxnValid(long txnid); - - /** - * Returns {@code true} if such base file can be used to materialize the snapshot represented by - * this {@code ValidTxnList}. - * @param txnid highest txn in a given base_xxxx file - */ - public boolean isValidBase(long txnid); - - /** - * Find out if a range of transaction ids are valid. Note that valid may have different meanings - * for different implementations, as some will only want to see committed transactions and some - * both committed and aborted. - * @param minTxnId minimum txnid to look for, inclusive - * @param maxTxnId maximum txnid to look for, inclusive - * @return Indicate whether none, some, or all of these transactions are valid. - */ - public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId); - - /** - * Write this validTxnList into a string. This should produce a string that - * can be used by {@link #readFromString(String)} to populate a validTxnsList. - */ - public String writeToString(); - - /** - * Populate this validTxnList from the string. It is assumed that the string - * was created via {@link #writeToString()} and the exceptions list is sorted. - * @param src source string. - */ - public void readFromString(String src); - - /** - * Get the largest transaction id used. - * @return largest transaction id used - */ - public long getHighWatermark(); - - /** - * Get the list of transactions under the high water mark that are not valid. Note that invalid - * may have different meanings for different implementations, as some will only want to see open - * transactions and some both open and aborted. - * @return a list of invalid transaction ids - */ - public long[] getInvalidTransactions(); - - /** - * Indicates whether a given transaction is aborted. - * @param txnid id for the transaction - * @return true if aborted, false otherwise - */ - public boolean isTxnAborted(long txnid); - - /** - * Find out if a range of transaction ids are aborted. - * @param minTxnId minimum txnid to look for, inclusive - * @param maxTxnId maximum txnid to look for, inclusive - * @return Indicate whether none, some, or all of these transactions are aborted. - */ - public RangeResponse isTxnRangeAborted(long minTxnId, long maxTxnId); - - /** - * Returns smallest Open transaction in this set, {@code null} if there is none. - */ - Long getMinOpenTxn(); -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java deleted file mode 100644 index 775233cb6e3cd091b570634a7e030e4352f24ea6..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.io; - -import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer; - -/** An allocator provided externally to storage classes to allocate MemoryBuffer-s. */ -public interface Allocator { - public static class AllocatorOutOfMemoryException extends RuntimeException { - public AllocatorOutOfMemoryException(String msg) { - super(msg); - } - - private static final long serialVersionUID = 268124648177151761L; - } - - public interface BufferObjectFactory { - MemoryBuffer create(); - } - - /** - * Allocates multiple buffers of a given size. - * @param dest Array where buffers are placed. Objects are reused if already there - * (see createUnallocated), created otherwise. - * @param size Allocation size. - * @throws AllocatorOutOfMemoryException Cannot allocate. - */ - @Deprecated - void allocateMultiple(MemoryBuffer[] dest, int size) throws AllocatorOutOfMemoryException; - - /** - * Allocates multiple buffers of a given size. - * @param dest Array where buffers are placed. Objects are reused if already there - * (see createUnallocated), created otherwise. - * @param size Allocation size. - * @param factory A factory to create the objects in the dest array, if needed. - * @throws AllocatorOutOfMemoryException Cannot allocate. - */ - void allocateMultiple(MemoryBuffer[] dest, int size, BufferObjectFactory factory) - throws AllocatorOutOfMemoryException; - - /** - * Creates an unallocated memory buffer object. This object can be passed to allocateMultiple - * to allocate; this is useful if data structures are created for separate buffers that can - * later be allocated together. - * @return a new unallocated memory buffer - */ - @Deprecated - MemoryBuffer createUnallocated(); - - /** Deallocates a memory buffer. - * @param buffer the buffer to deallocate - */ - void deallocate(MemoryBuffer buffer); - - /** Whether the allocator uses direct buffers. - * @return are they direct buffers? - */ - boolean isDirectAlloc(); - - /** Maximum allocation size supported by this allocator. - * @return a number of bytes - */ - int getMaxAllocation(); -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java deleted file mode 100644 index 552f20e276a6cd2d93ecc09292985c842ee22b87..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.io; - -import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer; - -/** An abstract data cache that IO formats can use to retrieve and cache data. */ -public interface DataCache { - public static final class BooleanRef { - public boolean value; - } - - /** Disk range factory used during cache retrieval. */ - public interface DiskRangeListFactory { - DiskRangeList createCacheChunk(MemoryBuffer buffer, long startOffset, long endOffset); - } - - /** - * Gets file data for particular offsets. The range list is modified in place; it is then - * returned (since the list head could have changed). Ranges are replaced with cached ranges. - * - * Any such buffer is locked in cache to prevent eviction, and must therefore be released - * back to cache via a corresponding call (releaseBuffer) when the caller is done with it. - * - * In case of partial overlap with cached data, full cache blocks are always returned; - * there's no capacity for partial matches in return type. The rules are as follows: - * 1) If the requested range starts in the middle of a cached range, that cached range will not - * be returned by default (e.g. if [100,200) and [200,300) are cached, the request for - * [150,300) will only return [200,300) from cache). This may be configurable in impls. - * This is because we assume well-known range start offsets are used (rg/stripe offsets), so - * a request from the middle of the start doesn't make sense. - * 2) If the requested range ends in the middle of a cached range, that entire cached range will - * be returned (e.g. if [100,200) and [200,300) are cached, the request for [100,250) will - * return both ranges). It should really be same as #1, however currently ORC uses estimated - * end offsets; if we don't return the end block, the caller may read it from disk needlessly. - * - * @param fileKey Unique ID of the target file on the file system. - * @param range A set of DiskRange-s (linked list) that is to be retrieved. May be modified. - * @param baseOffset base offset for the ranges (stripe/stream offset in case of ORC). - * @param factory A factory to produce DiskRangeList-s out of cached MemoryBuffer-s. - * @param gotAllData An out param - whether all the requested data was found in cache. - * @return The new or modified list of DiskRange-s, where some ranges may contain cached data. - */ - DiskRangeList getFileData(Object fileKey, DiskRangeList range, long baseOffset, - DiskRangeListFactory factory, BooleanRef gotAllData); - - /** - * Puts file data into cache, or gets older data in case of collisions. - * - * The memory buffers provided MUST be allocated via an allocator returned by getAllocator - * method, to allow cache implementations that evict and then de-allocate the buffer. - * - * It is assumed that the caller will use the data immediately, therefore any buffers provided - * to putFileData (or returned due to cache collision) are locked in cache to prevent eviction, - * and must therefore be released back to cache via a corresponding call (releaseBuffer) when the - * caller is done with it. Buffers rejected due to conflict will neither be locked, nor - * automatically deallocated. The caller must take care to discard these buffers. - * - * @param fileKey Unique ID of the target file on the file system. - * @param ranges The ranges for which the data is being cached. These objects will not be stored. - * @param data The data for the corresponding ranges. - * @param baseOffset base offset for the ranges (stripe/stream offset in case of ORC). - * @return null if all data was put; bitmask indicating which chunks were not put otherwise; - * the replacement chunks from cache are updated directly in the array. - */ - long[] putFileData(Object fileKey, DiskRange[] ranges, MemoryBuffer[] data, long baseOffset); - - /** - * Releases the buffer returned by getFileData/provided to putFileData back to cache. - * See respective javadocs for details. - * @param buffer the buffer to release - */ - void releaseBuffer(MemoryBuffer buffer); - - /** - * Notifies the cache that the buffer returned from getFileData/provided to putFileData will - * be used by another consumer and therefore released multiple times (one more time per call). - * @param buffer the buffer to reuse - */ - void reuseBuffer(MemoryBuffer buffer); - - /** - * Gets the allocator associated with this DataCache. - * @return the allocator - */ - Allocator getAllocator(); - - /** - * Gets the buffer object factory associated with this DataCache, to use with allocator. - * @return the factory - */ - Allocator.BufferObjectFactory getDataBufferFactory(); -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRange.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRange.java deleted file mode 100644 index 33aecf53e703b72482e4e29c19170bbdebc815e3..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRange.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.io; - -import java.nio.ByteBuffer; - -/** - * The sections of a file. - */ -public class DiskRange { - /** The first address. */ - protected long offset; - /** The address afterwards. */ - protected long end; - - public DiskRange(long offset, long end) { - this.offset = offset; - this.end = end; - if (end < offset) { - throw new IllegalArgumentException("invalid range " + this); - } - } - - @Override - public boolean equals(Object other) { - if (other == null || other.getClass() != getClass()) { - return false; - } - return equalRange((DiskRange) other); - } - - public boolean equalRange(DiskRange other) { - return other.offset == offset && other.end == end; - } - - @Override - public int hashCode() { - return (int)(offset ^ (offset >>> 32)) * 31 + (int)(end ^ (end >>> 32)); - } - - @Override - public String toString() { - return "range start: " + offset + " end: " + end; - } - - public long getOffset() { - return offset; - } - - public long getEnd() { - return end; - } - - public int getLength() { - long len = this.end - this.offset; - assert len <= Integer.MAX_VALUE; - return (int)len; - } - - // For subclasses - public boolean hasData() { - return false; - } - - public DiskRange sliceAndShift(long offset, long end, long shiftBy) { - // Rather, unexpected usage exception. - throw new UnsupportedOperationException(); - } - - public ByteBuffer getData() { - throw new UnsupportedOperationException(); - } - - protected boolean merge(long otherOffset, long otherEnd) { - if (!overlap(offset, end, otherOffset, otherEnd)) return false; - offset = Math.min(offset, otherOffset); - end = Math.max(end, otherEnd); - return true; - } - - private static boolean overlap(long leftA, long rightA, long leftB, long rightB) { - if (leftA <= leftB) { - return rightA >= leftB; - } - return rightB >= leftA; - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java deleted file mode 100644 index 13494b6e413aff1ba4c5a97bd3afa96e39d148a7..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java +++ /dev/null @@ -1,297 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.io; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Java linked list iterator interface is convoluted, and moreover concurrent modifications - * of the same list by multiple iterators are impossible. Hence, this. - * Java also doesn't support multiple inheritance, so this cannot be done as "aspect"... */ -public class DiskRangeList extends DiskRange { - private static final Logger LOG = LoggerFactory.getLogger(DiskRangeList.class); - public DiskRangeList prev, next; - - public DiskRangeList(long offset, long end) { - super(offset, end); - } - - /** Replaces this element with another in the list; returns the new element. - * @param other the disk range to swap into this list - * @return the new element - */ - public DiskRangeList replaceSelfWith(DiskRangeList other) { - checkArg(other); - other.prev = this.prev; - other.next = this.next; - if (this.prev != null) { - checkOrder(this.prev, other, this); - this.prev.next = other; - } - if (this.next != null) { - checkOrder(other, this.next, this); - this.next.prev = other; - } - this.next = this.prev = null; - return other; - } - - private final static void checkOrder(DiskRangeList prev, DiskRangeList next, DiskRangeList ref) { - if (prev.getEnd() <= next.getOffset()) return; - assertInvalidOrder(ref.prev == null ? ref : ref.prev, prev, next); - } - - private final static void assertInvalidOrder( - DiskRangeList ref, DiskRangeList prev, DiskRangeList next) { - String error = "Elements not in order " + prev + " and " + next - + "; trying to insert into " + stringifyDiskRanges(ref); - LOG.error(error); - throw new AssertionError(error); - } - - // TODO: this duplicates a method in ORC, but the method should actually be here. - public final static String stringifyDiskRanges(DiskRangeList range) { - StringBuilder buffer = new StringBuilder(); - buffer.append("["); - boolean isFirst = true; - while (range != null) { - if (!isFirst) { - buffer.append(", {"); - } else { - buffer.append("{"); - } - isFirst = false; - buffer.append(range.toString()); - buffer.append("}"); - range = range.next; - } - buffer.append("]"); - return buffer.toString(); - } - - private void checkArg(DiskRangeList other) { - if (other == this) { - // The only case where duplicate elements matter... the others are handled by the below. - throw new AssertionError("Inserting self into the list [" + other + "]"); - } - if (other.prev != null || other.next != null) { - throw new AssertionError("[" + other + "] is part of another list; prev [" - + other.prev + "], next [" + other.next + "]"); - } - } - - /** - * Inserts an intersecting range before current in the list and adjusts offset accordingly. - * @param other the element to insert - * @return the new element. - */ - public DiskRangeList insertPartBefore(DiskRangeList other) { - checkArg(other); - if (other.end <= this.offset || other.end > this.end) { - assertInvalidOrder(this.prev == null ? this : this.prev, other, this); - } - this.offset = other.end; - other.prev = this.prev; - other.next = this; - if (this.prev != null) { - checkOrder(this.prev, other, this.prev); - this.prev.next = other; - } - this.prev = other; - return other; - } - - /** - * Inserts an element after current in the list. - * @param other the new element to insert - * @return the new element. - * */ - public DiskRangeList insertAfter(DiskRangeList other) { - checkArg(other); - checkOrder(this, other, this); - return insertAfterInternal(other); - } - - private DiskRangeList insertAfterInternal(DiskRangeList other) { - other.next = this.next; - other.prev = this; - if (this.next != null) { - checkOrder(other, this.next, this); - this.next.prev = other; - } - this.next = other; - return other; - } - - /** - * Inserts an intersecting range after current in the list and adjusts offset accordingly. - * @param other the new element to insert - * @return the new element. - */ - public DiskRangeList insertPartAfter(DiskRangeList other) { - // The only allowed non-overlapping option is extra bytes at the end. - if (other.offset > this.end || other.offset <= this.offset || other.end <= this.offset) { - assertInvalidOrder(this.prev == null ? this : this.prev, this, other); - } - this.end = other.offset; - return insertAfter(other); - } - - /** Removes an element after current from the list. */ - public void removeAfter() { - DiskRangeList other = this.next; - if (this == other) { - throw new AssertionError("Invalid duplicate [" + other + "]"); - } - this.next = other.next; - if (this.next != null) { - this.next.prev = this; - } - other.next = other.prev = null; - } - - /** Removes the current element from the list. */ - public void removeSelf() { - if (this.prev == this || this.next == this) { - throw new AssertionError("Invalid duplicate [" + this + "]"); - } - if (this.prev != null) { - this.prev.next = this.next; - } - if (this.next != null) { - this.next.prev = this.prev; - } - this.next = this.prev = null; - } - - /** Splits current element in the list, using DiskRange::slice. - * @param cOffset the position to split the list - * @return the split list - */ - public final DiskRangeList split(long cOffset) { - DiskRangeList right = insertAfterInternal((DiskRangeList)this.sliceAndShift(cOffset, end, 0)); - DiskRangeList left = replaceSelfWith((DiskRangeList)this.sliceAndShift(offset, cOffset, 0)); - checkOrder(left, right, left); // Prev/next are already checked in the calls. - return left; - } - - public boolean hasContiguousNext() { - return next != null && end == next.offset; - } - - // @VisibleForTesting - public int listSize() { - int result = 1; - DiskRangeList current = this.next; - while (current != null) { - ++result; - current = current.next; - } - return result; - } - - public long getTotalLength() { - long totalLength = getLength(); - DiskRangeList current = next; - while (current != null) { - totalLength += current.getLength(); - current = current.next; - } - return totalLength; - } - - // @VisibleForTesting - public DiskRangeList[] listToArray() { - DiskRangeList[] result = new DiskRangeList[listSize()]; - int i = 0; - DiskRangeList current = this.next; - while (current != null) { - result[i] = current; - ++i; - current = current.next; - } - return result; - } - - public static class CreateHelper { - private DiskRangeList tail = null, head; - - public DiskRangeList getTail() { - return tail; - } - - public void addOrMerge(long offset, long end, boolean doMerge, boolean doLogNew) { - if (doMerge && tail != null && tail.merge(offset, end)) return; - if (doLogNew) { - LOG.debug("Creating new range; last range (which can include some previous adds) was " - + tail); - } - DiskRangeList node = new DiskRangeList(offset, end); - if (tail == null) { - head = tail = node; - } else { - tail = tail.insertAfter(node); - } - } - - public DiskRangeList get() { - return head; - } - - public DiskRangeList extract() { - DiskRangeList result = head; - head = null; - return result; - } - } - - /** - * List in-place mutation helper - a bogus first element that is inserted before list head, - * and thus remains constant even if head is replaced with some new range via in-place list - * mutation. extract() can be used to obtain the modified list. - */ - public static class MutateHelper extends DiskRangeList { - public MutateHelper(DiskRangeList head) { - super(-1, -1); - assert head != null; - assert head.prev == null; - this.next = head; - head.prev = this; - } - - public DiskRangeList get() { - return next; - } - - public DiskRangeList extract() { - DiskRangeList result = this.next; - assert result != null; - this.next = result.prev = null; - return result; - } - } - - public void setEnd(long newEnd) { - assert newEnd >= this.offset; - assert this.next == null || this.next.offset >= newEnd; - this.end = newEnd; - if (this.next != null) { - checkOrder(this, this.next, this); - } - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java deleted file mode 100644 index e684ece3edbf420e05adadc7a1f9599553ceb6d4..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.io; - - -import java.nio.ByteBuffer; -import java.io.IOException; -import java.io.InputStream; - -import org.apache.hadoop.hive.common.io.encoded.MemoryBufferOrBuffers; - -public interface FileMetadataCache { - /** - * @return Metadata for a given file (ORC or Parquet footer). - * The caller must decref this buffer when done. - */ - MemoryBufferOrBuffers getFileMetadata(Object fileKey); - - /** - * Puts the metadata for a given file (e.g. a footer buffer into cache). - * @param fileKey The file key. - * @param length The footer length. - * @param is The stream to read the footer from. - * @return The buffer or buffers representing the cached footer. - * The caller must decref this buffer when done. - */ - MemoryBufferOrBuffers putFileMetadata( - Object fileKey, int length, InputStream is) throws IOException; - - MemoryBufferOrBuffers putFileMetadata(Object fileKey, ByteBuffer tailBuffer); - - /** - * Releases the buffer returned from getFileMetadata or putFileMetadata method. - * @param buffer The buffer to release. - */ - void decRefBuffer(MemoryBufferOrBuffers buffer); -} \ No newline at end of file diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java deleted file mode 100644 index 63084b902a0cc684865f18a74621110480138643..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.io.encoded; - -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A block of data for a given section of a file, similar to VRB but in encoded form. - * Stores a set of buffers for each encoded stream that is a part of each column. - */ -public class EncodedColumnBatch { - /** - * Slice of the data for a stream for some column, stored inside MemoryBuffer's. - * ColumnStreamData can be reused for many EncodedColumnBatch-es (e.g. dictionary stream), so - * it tracks the number of such users via a refcount. - */ - public static class ColumnStreamData { - private List cacheBuffers; - /** Base offset from the beginning of the indexable unit; for example, for ORC, - * offset from the CB in a compressed file, from the stream in uncompressed file. */ - private int indexBaseOffset = 0; - - /** Reference count. */ - private AtomicInteger refCount = new AtomicInteger(0); - - public void reset() { - cacheBuffers.clear(); - refCount.set(0); - indexBaseOffset = 0; - } - - public void incRef() { - refCount.incrementAndGet(); - } - - public int decRef() { - int i = refCount.decrementAndGet(); - assert i >= 0; - return i; - } - - public List getCacheBuffers() { - return cacheBuffers; - } - - public void setCacheBuffers(List cacheBuffers) { - this.cacheBuffers = cacheBuffers; - } - - public int getIndexBaseOffset() { - return indexBaseOffset; - } - - public void setIndexBaseOffset(int indexBaseOffset) { - this.indexBaseOffset = indexBaseOffset; - } - - @Override - public String toString() { - String bufStr = ""; - if (cacheBuffers != null) { - for (MemoryBuffer mb : cacheBuffers) { - bufStr += mb.getClass().getSimpleName() + " with " + mb.getByteBufferRaw().remaining() + " bytes, "; - } - } - return "ColumnStreamData [cacheBuffers=[" + bufStr - + "], indexBaseOffset=" + indexBaseOffset + "]"; - } - - } - - /** The key that is used to map this batch to source location. */ - protected BatchKey batchKey; - /** - * Stream data for each column that has true in the corresponding hasData position. - * For each column, streams are indexed by kind (for ORC), with missing elements being null. - */ - protected ColumnStreamData[][] columnData; - /** Indicates which columns have data. Correspond to columnData elements. */ - protected boolean[] hasData; - - public void reset() { - if (hasData != null) { - Arrays.fill(hasData, false); - } - if (columnData == null) return; - for (int i = 0; i < columnData.length; ++i) { - if (columnData[i] == null) continue; - for (int j = 0; j < columnData[i].length; ++j) { - columnData[i][j] = null; - } - } - } - - - public void initColumn(int colIx, int streamCount) { - hasData[colIx] = true; - if (columnData[colIx] == null || columnData[colIx].length != streamCount) { - columnData[colIx] = new ColumnStreamData[streamCount]; - } - } - - private static final Logger LOG = LoggerFactory.getLogger(EncodedColumnBatch.class); - public void setStreamData(int colIx, int streamIx, ColumnStreamData csd) { - assert hasData[colIx]; - columnData[colIx][streamIx] = csd; - } - - public BatchKey getBatchKey() { - return batchKey; - } - - public ColumnStreamData[] getColumnData(int colIx) { - if (!hasData[colIx]) throw new AssertionError("No data for column " + colIx); - return columnData[colIx]; - } - - public int getTotalColCount() { - return columnData.length; // Includes the columns that have no data - } - - protected void resetColumnArrays(int columnCount) { - if (hasData != null && columnCount == hasData.length) { - Arrays.fill(hasData, false); - return; - } - hasData = new boolean[columnCount]; - ColumnStreamData[][] columnData = new ColumnStreamData[columnCount][]; - if (this.columnData != null) { - for (int i = 0; i < Math.min(columnData.length, this.columnData.length); ++i) { - columnData[i] = this.columnData[i]; - } - } - this.columnData = columnData; - } - - public boolean hasData(int colIx) { - return hasData[colIx]; - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java deleted file mode 100644 index 201c5e2bc4ee36b1249e687429f22ced9d77c94c..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.io.encoded; - -import java.nio.ByteBuffer; - -/** Abstract interface for any class wrapping a ByteBuffer. */ -public interface MemoryBuffer { - /** - * Get the raw byte buffer that backs this buffer. - * Note - raw buffer should not be modified. - * @return the shared byte buffer - */ - public ByteBuffer getByteBufferRaw(); - public ByteBuffer getByteBufferDup(); -} \ No newline at end of file diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBufferOrBuffers.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBufferOrBuffers.java deleted file mode 100644 index c04310a2ba9390a29e86673002e199edff37ba91..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBufferOrBuffers.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.io.encoded; - -public interface MemoryBufferOrBuffers { - MemoryBuffer getSingleBuffer(); - MemoryBuffer[] getMultipleBuffers(); -} \ No newline at end of file diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/DataTypePhysicalVariation.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/DataTypePhysicalVariation.java deleted file mode 100644 index 778c8c3d793092780b56101ca0054502d7a8f5cc..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/type/DataTypePhysicalVariation.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -public enum DataTypePhysicalVariation { - NONE, - DECIMAL_64 -} \ No newline at end of file diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java deleted file mode 100644 index 4484ed27b5a432f96451db0f94a6553074a6dd2e..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java +++ /dev/null @@ -1,759 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.math.BigDecimal; -import java.math.BigInteger; - -/** - * FastHiveDecimal is a mutable fast decimal object. It is the base class for both the - * HiveDecimal and HiveDecimalWritable classes. All fast* methods are protected so they - * cannot be accessed by clients of HiveDecimal and HiveDecimalWritable. HiveDecimal ensures - * it creates new objects when the value changes since it provides immutable semantics; - * HiveDecimalWritable does not create new objects since it provides mutable semantics. - * - * The methods in this class are shells that pickup the member variables from FastHiveDecimal - * parameters and pass them as individual parameters to static methods in the FastHiveDecimalImpl - * class that do the real work. - * - * NOTE: The rationale for fast decimal is in FastHiveDecimalImpl. - */ -public class FastHiveDecimal { - - /* - * We use protected for the fields so the FastHiveDecimalImpl class can access them. Other - * classes including HiveDecimal should not access these fields directly. - */ - - // See FastHiveDecimalImpl for more details on these fields. - - // -1 when negative; 0 when decimal is zero; 1 when positive. - protected int fastSignum; - - // Decimal longwords. - protected long fast2; - protected long fast1; - protected long fast0; - - // The number of integer digits in the decimal. When the integer portion is zero, this is 0. - protected int fastIntegerDigitCount; - - // The scale of the decimal. - protected int fastScale; - - // Used for legacy HiveDecimalV1 setScale compatibility for binary / display serialization of - // trailing zeroes (or rounding). - protected int fastSerializationScale; - - protected FastHiveDecimal() { - fastReset(); - } - - protected FastHiveDecimal(FastHiveDecimal fastDec) { - this(); - fastSignum = fastDec.fastSignum; - fast0 = fastDec.fast0; - fast1 = fastDec.fast1; - fast2 = fastDec.fast2; - fastIntegerDigitCount = fastDec.fastIntegerDigitCount; - fastScale = fastDec.fastScale; - - // Not propagated. - fastSerializationScale = -1; - } - - protected FastHiveDecimal(int fastSignum, FastHiveDecimal fastDec) { - this(); - this.fastSignum = fastSignum; - fast0 = fastDec.fast0; - fast1 = fastDec.fast1; - fast2 = fastDec.fast2; - fastIntegerDigitCount = fastDec.fastIntegerDigitCount; - fastScale = fastDec.fastScale; - - // Not propagated. - fastSerializationScale = -1; - } - - protected FastHiveDecimal( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - this(); - this.fastSignum = fastSignum; - this.fast0 = fast0; - this.fast1 = fast1; - this.fast2 = fast2; - this.fastIntegerDigitCount = fastIntegerDigitCount; - this.fastScale = fastScale; - - fastSerializationScale = -1; - } - - protected FastHiveDecimal(long longValue) { - this(); - FastHiveDecimalImpl.fastSetFromLong(longValue, this); - } - - protected FastHiveDecimal(String string) { - this(); - FastHiveDecimalImpl.fastSetFromString(string, false, this); - } - - protected void fastReset() { - fastSignum = 0; - fast0 = 0; - fast1 = 0; - fast2 = 0; - fastIntegerDigitCount = 0; - fastScale = 0; - fastSerializationScale = -1; - } - - protected void fastSet(FastHiveDecimal fastDec) { - fastSignum = fastDec.fastSignum; - fast0 = fastDec.fast0; - fast1 = fastDec.fast1; - fast2 = fastDec.fast2; - fastIntegerDigitCount = fastDec.fastIntegerDigitCount; - fastScale = fastDec.fastScale; - fastSerializationScale = fastDec.fastSerializationScale; - } - - protected void fastSet( - int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) { - this.fastSignum = fastSignum; - this.fast0 = fast0; - this.fast1 = fast1; - this.fast2 = fast2; - this.fastIntegerDigitCount = fastIntegerDigitCount; - this.fastScale = fastScale; - - // Not specified. - fastSerializationScale = -1; - } - - protected void fastSetSerializationScale(int fastSerializationScale) { - this.fastSerializationScale = fastSerializationScale; - } - - protected int fastSerializationScale() { - return fastSerializationScale; - } - - protected static final String STRING_ENFORCE_PRECISION_OUT_OF_RANGE = - "Decimal precision out of allowed range [1," + HiveDecimal.MAX_PRECISION + "]"; - protected static final String STRING_ENFORCE_SCALE_OUT_OF_RANGE = - "Decimal scale out of allowed range [0," + HiveDecimal.MAX_SCALE + "]"; - protected static final String STRING_ENFORCE_SCALE_LESS_THAN_EQUAL_PRECISION = - "Decimal scale must be less than or equal to precision"; - - protected boolean fastSetFromBigDecimal( - BigDecimal bigDecimal, boolean allowRounding) { - return - FastHiveDecimalImpl.fastSetFromBigDecimal( - bigDecimal, allowRounding, this); - } - - protected boolean fastSetFromBigInteger( - BigInteger bigInteger) { - return - FastHiveDecimalImpl.fastSetFromBigInteger( - bigInteger, this); - } - - protected boolean fastSetFromBigIntegerAndScale( - BigInteger bigInteger, int scale) { - return - FastHiveDecimalImpl.fastSetFromBigInteger( - bigInteger, scale, this); - } - - protected boolean fastSetFromString(String string, boolean trimBlanks) { - byte[] bytes = string.getBytes(); - return - fastSetFromBytes( - bytes, 0, bytes.length, trimBlanks); - } - - protected boolean fastSetFromBytes(byte[] bytes, int offset, int length, boolean trimBlanks) { - return - FastHiveDecimalImpl.fastSetFromBytes( - bytes, offset, length, trimBlanks, this); - } - - protected boolean fastSetFromDigitsOnlyBytesAndScale( - boolean isNegative, byte[] bytes, int offset, int length, int scale) { - return - FastHiveDecimalImpl.fastSetFromDigitsOnlyBytesAndScale( - isNegative, bytes, offset, length, scale, this); - } - - protected void fastSetFromInt(int intValue) { - FastHiveDecimalImpl.fastSetFromInt(intValue, this); - } - - protected void fastSetFromLong(long longValue) { - FastHiveDecimalImpl.fastSetFromLong(longValue, this); - } - - protected boolean fastSetFromLongAndScale(long longValue, int scale) { - return - FastHiveDecimalImpl.fastSetFromLongAndScale( - longValue, scale, this); - } - - protected boolean fastSetFromFloat(float floatValue) { - return - FastHiveDecimalImpl.fastSetFromFloat( - floatValue, this); - } - - protected boolean fastSetFromDouble(double doubleValue) { - return - FastHiveDecimalImpl.fastSetFromDouble( - doubleValue, this); - } - - protected void fastFractionPortion() { - FastHiveDecimalImpl.fastFractionPortion( - fastSignum, fast0, fast1, fast2, fastScale, - this); - } - - protected void fastIntegerPortion() { - FastHiveDecimalImpl.fastIntegerPortion( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale, - this); - } - - protected static final int FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ = 8 * 3; - - protected boolean fastSerializationUtilsRead( - InputStream inputStream, int scale, - byte[] scratchBytes) throws IOException, EOFException { - return - FastHiveDecimalImpl.fastSerializationUtilsRead( - inputStream, scale, scratchBytes, this); - } - - protected boolean fastSetFromBigIntegerBytesAndScale( - byte[] bytes, int offset, int length, int scale) { - return FastHiveDecimalImpl.fastSetFromBigIntegerBytesAndScale( - bytes, offset, length, scale, this); - } - - protected static final int SCRATCH_LONGS_LEN_FAST_SERIALIZATION_UTILS_WRITE = 6; - - protected boolean fastSerializationUtilsWrite(OutputStream outputStream, - long[] scratchLongs) - throws IOException { - return - FastHiveDecimalImpl.fastSerializationUtilsWrite( - outputStream, - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale, - scratchLongs); - } - - /* - * Deserializes 64-bit decimals up to the maximum 64-bit precision (18 decimal digits). - */ - protected void fastDeserialize64(long decimalLong, int scale) { - FastHiveDecimalImpl.fastDeserialize64( - decimalLong, scale, this); - } - - /* - * Serializes decimal64 up to the maximum 64-bit precision (18 decimal digits). - */ - protected long fastSerialize64(int scale) { - return - FastHiveDecimalImpl.fastSerialize64( - scale, - fastSignum, fast1, fast0, fastScale); - } - - // The fastBigIntegerBytes method returns 3 56 bit (7 byte) words and a possible sign byte. - // However, the fastBigIntegerBytes can take on trailing zeroes -- so make it larger. - protected static final int FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES = 1 + 48; - protected static final int FAST_SCRATCH_LONGS_LEN = 6; - - protected int fastBigIntegerBytes( - long[] scratchLongs, byte[] buffer) { - return - FastHiveDecimalImpl.fastBigIntegerBytes( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - fastSerializationScale, - scratchLongs, buffer); - } - - protected int fastBigIntegerBytesScaled( - int serializationScale, - long[] scratchLongs, byte[] buffer) { - return - FastHiveDecimalImpl.fastBigIntegerBytesScaled( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - serializationScale, - scratchLongs, buffer); - } - - protected boolean fastIsByte() { - return - FastHiveDecimalImpl.fastIsByte( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - } - - protected byte fastByteValueClip() { - return - FastHiveDecimalImpl.fastByteValueClip( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - } - - protected boolean fastIsShort() { - return - FastHiveDecimalImpl.fastIsShort( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - } - - protected short fastShortValueClip() { - return - FastHiveDecimalImpl.fastShortValueClip( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - } - - protected boolean fastIsInt() { - return - FastHiveDecimalImpl.fastIsInt( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - } - - protected int fastIntValueClip() { - return - FastHiveDecimalImpl.fastIntValueClip( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - } - - protected boolean fastIsLong() { - return - FastHiveDecimalImpl.fastIsLong( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - } - - protected long fastLongValueClip() { - return - FastHiveDecimalImpl.fastLongValueClip( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - } - - protected float fastFloatValue() { - return - FastHiveDecimalImpl.fastFloatValue( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - } - - protected double fastDoubleValue() { - return - FastHiveDecimalImpl.fastDoubleValue( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - } - - protected BigInteger fastBigIntegerValue() { - return - FastHiveDecimalImpl.fastBigIntegerValue( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - fastSerializationScale); - } - - protected BigDecimal fastBigDecimalValue() { - return - FastHiveDecimalImpl.fastBigDecimalValue( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale); - } - - protected int fastScale() { - return fastScale; - } - - protected int fastSignum() { - return fastSignum; - } - - protected int fastCompareTo(FastHiveDecimal right) { - return - FastHiveDecimalImpl.fastCompareTo( - fastSignum, fast0, fast1, fast2, - fastScale, - right.fastSignum, right.fast0, right.fast1, right.fast2, - right.fastScale); - } - - protected static int fastCompareTo(FastHiveDecimal left, FastHiveDecimal right) { - return - FastHiveDecimalImpl.fastCompareTo( - left.fastSignum, left.fast0, left.fast1, left.fast2, - left.fastScale, - right.fastSignum, right.fast0, right.fast1, right.fast2, - right.fastScale); - } - - protected boolean fastEquals(FastHiveDecimal that) { - return - FastHiveDecimalImpl.fastEquals( - fastSignum, fast0, fast1, fast2, - fastScale, - that.fastSignum, that.fast0, that.fast1, that.fast2, - that.fastScale); - } - - protected void fastAbs() { - fastSignum = 1; - } - - protected void fastNegate() { - if (fastSignum == 0) { - return; - } - fastSignum = (fastSignum == 1 ? -1 : 1); - } - - protected int fastNewFasterHashCode() { - return - FastHiveDecimalImpl.fastNewFasterHashCode( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - } - - protected int fastHashCode() { - return - FastHiveDecimalImpl.fastHashCode( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - } - - protected int fastIntegerDigitCount() { - return fastIntegerDigitCount; - } - - protected int fastSqlPrecision() { - return - FastHiveDecimalImpl.fastSqlPrecision( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale); - } - - protected int fastRawPrecision() { - return - FastHiveDecimalImpl.fastRawPrecision( - fastSignum, fast0, fast1, fast2); - } - - protected boolean fastScaleByPowerOfTen( - int n, - FastHiveDecimal fastResult) { - return - FastHiveDecimalImpl.fastScaleByPowerOfTen( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale, - n, - fastResult); - } - - protected static String fastRoundingModeToString(int roundingMode) { - String roundingModeString; - switch (roundingMode) { - case BigDecimal.ROUND_DOWN: - roundingModeString = "ROUND_DOWN"; - break; - case BigDecimal.ROUND_UP: - roundingModeString = "ROUND_UP"; - break; - case BigDecimal.ROUND_FLOOR: - roundingModeString = "ROUND_FLOOR"; - break; - case BigDecimal.ROUND_CEILING: - roundingModeString = "ROUND_CEILING"; - break; - case BigDecimal.ROUND_HALF_UP: - roundingModeString = "ROUND_HALF_UP"; - break; - case BigDecimal.ROUND_HALF_EVEN: - roundingModeString = "ROUND_HALF_EVEN"; - break; - default: - roundingModeString = "Unknown"; - } - return roundingModeString + " (" + roundingMode + ")"; - } - - protected boolean fastRound( - int newScale, int roundingMode, - FastHiveDecimal fastResult) { - return - FastHiveDecimalImpl.fastRound( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - newScale, roundingMode, - fastResult); - } - - protected boolean isAllZeroesBelow( - int power) { - return - FastHiveDecimalImpl.isAllZeroesBelow( - fastSignum, fast0, fast1, fast2, power); - } - - protected boolean fastEnforcePrecisionScale( - int maxPrecision, int maxScale) { - if (maxPrecision <= 0 || maxPrecision > HiveDecimal.MAX_PRECISION) { - return false; - } - if (maxScale < 0 || maxScale > HiveDecimal.MAX_SCALE) { - return false; - } - /* - if (!fastIsValid()) { - fastRaiseInvalidException(); - } - */ - FastCheckPrecisionScaleStatus status = - FastHiveDecimalImpl.fastCheckPrecisionScale( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - maxPrecision, maxScale); - switch (status) { - case NO_CHANGE: - return true; - case OVERFLOW: - return false; - case UPDATE_SCALE_DOWN: - { - if (!FastHiveDecimalImpl.fastUpdatePrecisionScale( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - maxPrecision, maxScale, status, - this)) { - return false; - } - /* - if (!fastIsValid()) { - fastRaiseInvalidException(); - } - */ - return true; - } - default: - throw new RuntimeException("Unknown fast decimal check precision and scale status " + status); - } - } - - protected FastCheckPrecisionScaleStatus fastCheckPrecisionScale( - int maxPrecision, int maxScale) { - return - FastHiveDecimalImpl.fastCheckPrecisionScale( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - maxPrecision, maxScale); - } - - protected static enum FastCheckPrecisionScaleStatus { - NO_CHANGE, - OVERFLOW, - UPDATE_SCALE_DOWN; - } - - protected boolean fastUpdatePrecisionScale( - int maxPrecision, int maxScale, FastCheckPrecisionScaleStatus status, - FastHiveDecimal fastResult) { - return - FastHiveDecimalImpl.fastUpdatePrecisionScale( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - maxPrecision, maxScale, status, - fastResult); - } - - protected boolean fastAdd( - FastHiveDecimal fastRight, - FastHiveDecimal fastResult) { - return - FastHiveDecimalImpl.fastAdd( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, - fastRight.fastIntegerDigitCount, fastRight.fastScale, - fastResult); - } - - protected boolean fastSubtract( - FastHiveDecimal fastRight, - FastHiveDecimal fastResult) { - return - FastHiveDecimalImpl.fastSubtract( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, - fastRight.fastIntegerDigitCount, fastRight.fastScale, - fastResult); - } - - protected boolean fastMultiply( - FastHiveDecimal fastRight, - FastHiveDecimal fastResult) { - return - FastHiveDecimalImpl.fastMultiply( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, - fastRight.fastIntegerDigitCount, fastRight.fastScale, - fastResult); - } - - protected boolean fastRemainder( - FastHiveDecimal fastRight, - FastHiveDecimal fastResult) { - return - FastHiveDecimalImpl.fastRemainder( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, - fastRight.fastIntegerDigitCount, fastRight.fastScale, - fastResult); - } - - protected boolean fastDivide( - FastHiveDecimal fastRight, - FastHiveDecimal fastResult) { - return - FastHiveDecimalImpl.fastDivide( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, - fastRight.fastIntegerDigitCount, fastRight.fastScale, - fastResult); - } - - protected boolean fastPow( - int exponent, - FastHiveDecimal fastResult) { - return - FastHiveDecimalImpl.fastPow( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale, - exponent, - fastResult); - } - - protected String fastToString( - byte[] scratchBuffer) { - return - FastHiveDecimalImpl.fastToString( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, -1, - scratchBuffer); - } - - protected String fastToString() { - return - FastHiveDecimalImpl.fastToString( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, -1); - } - - protected String fastToFormatString(int formatScale) { - return - FastHiveDecimalImpl.fastToFormatString( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - formatScale); - } - - protected String fastToFormatString( - int formatScale, - byte[] scratchBuffer) { - return - FastHiveDecimalImpl.fastToFormatString( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - formatScale, - scratchBuffer); - } - - protected String fastToDigitsOnlyString() { - return - FastHiveDecimalImpl.fastToDigitsOnlyString( - fast0, fast1, fast2, - fastIntegerDigitCount); - } - - // Sign, zero, dot, 2 * digits (to support toFormatString which can add a lot of trailing zeroes). - protected final static int FAST_SCRATCH_BUFFER_LEN_TO_BYTES = - 1 + 1 + 1 + 2 * FastHiveDecimalImpl.MAX_DECIMAL_DIGITS; - - protected int fastToBytes( - byte[] scratchBuffer) { - return - FastHiveDecimalImpl.fastToBytes( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, -1, - scratchBuffer); - } - - protected int fastToFormatBytes( - int formatScale, - byte[] scratchBuffer) { - return - FastHiveDecimalImpl.fastToFormatBytes( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - formatScale, - scratchBuffer); - } - - protected int fastToDigitsOnlyBytes( - byte[] scratchBuffer) { - return - FastHiveDecimalImpl.fastToDigitsOnlyBytes( - fast0, fast1, fast2, - fastIntegerDigitCount, - scratchBuffer); - } - - @Override - public String toString() { - return - FastHiveDecimalImpl.fastToString( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, -1); - } - - protected boolean fastIsValid() { - return FastHiveDecimalImpl.fastIsValid(this); - } - - protected void fastRaiseInvalidException() { - FastHiveDecimalImpl.fastRaiseInvalidException(this); - } - - protected void fastRaiseInvalidException(String parameters) { - FastHiveDecimalImpl.fastRaiseInvalidException(this, parameters); - } -} \ No newline at end of file diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java deleted file mode 100644 index 84ff1c64f5c46adf8a82377ce3e6215c1370135c..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java +++ /dev/null @@ -1,9466 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -import java.util.Arrays; -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.math.RoundingMode; - -/** - * This class is a companion to the FastHiveDecimal class that separates the essential of code - * out of FastHiveDecimal into static methods in this class so that they can be used directly - * by vectorization to implement decimals by storing the fast0, fast1, and fast2 longs and - * the fastSignum, fastScale, etc ints in the DecimalColumnVector class. - */ -public class FastHiveDecimalImpl extends FastHiveDecimal { - - /** - * Representation of fast decimals. - * - * We use 3 long words to store the 38 digits of fast decimals and and 3 integers for sign, - * integer digit count, and scale. - * - * The lower and middle long words store 16 decimal digits each; the high long word has - * 6 decimal digits; total 38 decimal digits. - * - * We do not try and represent fast decimal value as an unsigned 128 bit binary number in 2 longs. - * There are several important reasons for this. - * - * The effort to represent an unsigned 128 integer in 2 Java signed longs is very difficult, - * error prone, hard to debug, and not worth the effort. - * - * The focus here is on reusing memory (i.e. with HiveDecimalWritable) as often as possible. - * Reusing memory is good for grouping of fast decimal objects and related objects in CPU cache - * lines for fast memory access and eliminating the cost of allocating temporary objects and - * reducing the global cost of garbage collection. - * - * In other words, we are focused on avoiding the poor performance of Java general immutable - * objects. - * - * Reducing memory size or being concerned about the memory size of using 3 longs vs. 2 longs - * for 128 unsigned bits is not the focus here. - * - * Besides focusing on reusing memory, storing a limited number (16) decimal digits in the longs - * rather than compacting the value into all binary bits of 2 longs has a surprising benefit. - * - * One big part of implementing decimals turns out to be manipulating decimal digits. - * - * For example, rounding a decimal involves trimming off lower digits or clearing lower digits. - * Since radix 10 digits cannot be masked with binary masks, we use division and multiplication - * using powers of 10. We can easily manipulate the decimal digits in a long word using simple - * integer multiplication / division without doing emulated 128 binary bit multiplication / - * division (e.g. the defunct Decimal128 class). - * - * For example, say we want to scale (round) down the fraction digits of a decimal. - * - * final long divideFactor = powerOfTenTable[scaleDown]; - * final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; - * - * result0 = - * fast0 / divideFactor - * + ((fast1 % divideFactor) * multiplyFactor); - * result1 = - * fast1 / divideFactor - * + ((fast2 % divideFactor) * multiplyFactor); - * result2 = - * fast2 / divideFactor; - * - * It also turns out to do addition and subtraction of decimals with different scales can involve - * overlap using more than 3 long words. Manipulating extra words is a natural extension of - * the existing techniques. - * - * Why is the decimal digits representation easier to debug? You can see the decimal digits in - * the 3 long words and do not have to convert binary words to decimal to see the value. - * - * 16 decimal digits for a long was choose so that an int can have 1/2 or 8 decimal digits during - * multiplication of int half words so intermediate multiplication results do not overflow a long. - * And, so addition overflow is well below the sign bit of a long. - */ - - // Code Sections: - // Initialize (fastSetFrom*). - // Take Integer or Fractional Portion. - // Binary to Decimal Conversion. - // Decimal to Binary Conversion.r - // Emulate SerializationUtils Deserialization used by ORC. - // Emulate SerializationUtils Serialization used by ORC. - // Emulate BigInteger Deserialization used by LazyBinary and others. - // Emulate BigInteger Serialization used by LazyBinary and others. - // Decimal to Integer Conversion. - // Decimal to Non-Integer Conversion. - // Decimal Comparison. - // Decimal Rounding. - // Decimal Scale Up/Down. - // Decimal Precision / Trailing Zeroes. - // Decimal Addition / Subtraction. - // Decimal Multiply. - // Decimal Division / Remainder. - // Decimal String Formatting. - // Decimal Validation. - // Decimal Debugging. - - private static final long[] powerOfTenTable = { - 1L, // 0 - 10L, - 100L, - 1000L, - 10000L, - 100000L, - 1000000L, - 10000000L, - 100000000L, // 8 - 1000000000L, - 10000000000L, - 100000000000L, - 1000000000000L, - 10000000000000L, - 100000000000000L, - 1000000000000000L, - 10000000000000000L, // 16 - 100000000000000000L, - 1000000000000000000L, // 18 - }; - - public static final int MAX_DECIMAL_DIGITS = 38; - - /** - * Int: 8 decimal digits. An even number and 1/2 of MAX_LONGWORD_DECIMAL. - */ - private static final int INTWORD_DECIMAL_DIGITS = 8; - private static final int MULTIPLER_INTWORD_DECIMAL = (int) powerOfTenTable[INTWORD_DECIMAL_DIGITS]; - - /** - * Long: 16 decimal digits. An even number and twice MAX_INTWORD_DECIMAL. - */ - private static final int LONGWORD_DECIMAL_DIGITS = 16; - private static final long MAX_LONGWORD_DECIMAL = powerOfTenTable[LONGWORD_DECIMAL_DIGITS] - 1; - private static final long MULTIPLER_LONGWORD_DECIMAL = powerOfTenTable[LONGWORD_DECIMAL_DIGITS]; - - public static final int DECIMAL64_DECIMAL_DIGITS = 18; - public static final long MAX_ABS_DECIMAL64 = 999999999999999999L; // 18 9's -- quite reliable! - - private static final int TWO_X_LONGWORD_DECIMAL_DIGITS = 2 * LONGWORD_DECIMAL_DIGITS; - private static final int THREE_X_LONGWORD_DECIMAL_DIGITS = 3 * LONGWORD_DECIMAL_DIGITS; - private static final int FOUR_X_LONGWORD_DECIMAL_DIGITS = 4 * LONGWORD_DECIMAL_DIGITS; - - // 38 decimal maximum - 32 digits in 2 lower longs (6 digits here). - private static final int HIGHWORD_DECIMAL_DIGITS = MAX_DECIMAL_DIGITS - TWO_X_LONGWORD_DECIMAL_DIGITS; - private static final long MAX_HIGHWORD_DECIMAL = - powerOfTenTable[HIGHWORD_DECIMAL_DIGITS] - 1; - - // 38 * 2 or 76 full decimal maximum - (64 + 8) digits in 4 lower longs (4 digits here). - private static final long FULL_MAX_HIGHWORD_DECIMAL = - powerOfTenTable[MAX_DECIMAL_DIGITS * 2 - (FOUR_X_LONGWORD_DECIMAL_DIGITS + INTWORD_DECIMAL_DIGITS)] - 1; - - /** - * BigInteger constants. - */ - - private static final BigInteger BIG_INTEGER_TWO = BigInteger.valueOf(2); - private static final BigInteger BIG_INTEGER_FIVE = BigInteger.valueOf(5); - private static final BigInteger BIG_INTEGER_TEN = BigInteger.valueOf(10); - - public static final BigInteger BIG_INTEGER_MAX_DECIMAL = - BIG_INTEGER_TEN.pow(MAX_DECIMAL_DIGITS).subtract(BigInteger.ONE); - - private static final BigInteger BIG_INTEGER_MAX_LONGWORD_DECIMAL = - BigInteger.valueOf(MAX_LONGWORD_DECIMAL); - - private static final BigInteger BIG_INTEGER_LONGWORD_MULTIPLIER = - BigInteger.ONE.add(BIG_INTEGER_MAX_LONGWORD_DECIMAL); - private static final BigInteger BIG_INTEGER_LONGWORD_MULTIPLIER_2X = - BIG_INTEGER_LONGWORD_MULTIPLIER.multiply(BIG_INTEGER_LONGWORD_MULTIPLIER); - private static final BigInteger BIG_INTEGER_MAX_HIGHWORD_DECIMAL = - BigInteger.valueOf(MAX_HIGHWORD_DECIMAL); - private static final BigInteger BIG_INTEGER_HIGHWORD_MULTIPLIER = - BigInteger.ONE.add(BIG_INTEGER_MAX_HIGHWORD_DECIMAL); - - // UTF-8 byte constants used by string/UTF-8 bytes to decimal and decimal to String/UTF-8 byte - // conversion. - - // There is only one blank in UTF-8. - private static final byte BYTE_BLANK = (byte) ' '; - - private static final byte BYTE_DIGIT_ZERO = (byte) '0'; - private static final byte BYTE_DIGIT_NINE = (byte) '9'; - - // Decimal point. - private static final byte BYTE_DOT = (byte) '.'; - - // Sign. - private static final byte BYTE_MINUS = (byte) '-'; - private static final byte BYTE_PLUS = (byte) '+'; - - // Exponent E or e. - private static final byte BYTE_EXPONENT_LOWER = (byte) 'e'; - private static final byte BYTE_EXPONENT_UPPER = (byte) 'E'; - - //************************************************************************************************ - // Initialize (fastSetFrom*). - - /* - * All of the fastSetFrom* methods require the caller to pass a fastResult parameter has been - * reset for better performance. - */ - - private static void doRaiseSetFromBytesInvalid( - byte[] bytes, int offset, int length, - FastHiveDecimal fastResult) { - final int end = offset + length; - throw new RuntimeException( - "Invalid fast decimal \"" + - new String(bytes, offset, end) + "\"" + - " fastSignum " + fastResult.fastSignum + " fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2 + - " fastIntegerDigitCount " + fastResult.fastIntegerDigitCount +" fastScale " + fastResult.fastScale + - " stack trace: " + getStackTraceAsSingleLine(Thread.currentThread().getStackTrace())); - } - - /** - * Scan a byte array slice for a decimal number in UTF-8 bytes. - * - * Syntax: - * [+|-][integerPortion][.[fractionalDigits]][{E|e}[+|-]exponent] - * // Where at least one integer or fractional - * // digit is required... - * - * We handle too many fractional digits by doing rounding ROUND_HALF_UP. - * - * NOTE: The fastSetFromBytes method requires the caller to pass a fastResult parameter has been - * reset for better performance. - * - * @param bytes the bytes to copy from - * @param offset the starting location in bytes - * @param length the number of bytes to use from bytes - * @param trimBlanks should spaces be trimmed? - * @param fastResult True if the byte array slice was successfully converted to a decimal. - * @return Was a valid number found? - */ - public static boolean fastSetFromBytes(byte[] bytes, int offset, int length, boolean trimBlanks, - FastHiveDecimal fastResult) { - - final int bytesLength = bytes.length; - - if (offset < 0 || offset >= bytesLength) { - return false; - } - final int end = offset + length; - if (end <= offset || end > bytesLength) { - return false; - } - - // We start here with at least one byte. - int index = offset; - - if (trimBlanks) { - while (bytes[index] == BYTE_BLANK) { - if (++index >= end) { - return false; - } - } - } - - // Started with a few ideas from BigDecimal(char[] in, int offset, int len) constructor... - // But soon became very fast decimal specific. - - boolean isNegative = false; - if (bytes[index] == BYTE_MINUS) { - isNegative = true; - if (++index >= end) { - return false; - } - } else if (bytes[index] == BYTE_PLUS) { - if (++index >= end) { - return false; - } - } - - int precision = 0; - - // We fill starting with highest digit in highest longword (HIGHWORD_DECIMAL_DIGITS) and - // move down. At end will will shift everything down if necessary. - - int longWordIndex = 0; // Where 0 is the highest longword; 1 is middle longword, etc. - - int digitNum = HIGHWORD_DECIMAL_DIGITS; - long multiplier = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1]; - - int digitValue = 0; - long longWord = 0; - - long fast0 = 0; - long fast1 = 0; - long fast2 = 0; - - byte work; - - // Parse integer portion. - - boolean haveInteger = false; - while (true) { - work = bytes[index]; - if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) { - break; - } - haveInteger = true; - if (precision == 0 && work == BYTE_DIGIT_ZERO) { - // Ignore leading zeroes. - if (++index >= end) { - break; - } - continue; - } - digitValue = work - BYTE_DIGIT_ZERO; - if (digitNum == 0) { - - // Integer parsing move to next lower longword. - - // Save previous longword. - if (longWordIndex == 0) { - fast2 = longWord; - } else if (longWordIndex == 1) { - fast1 = longWord; - } else if (longWordIndex == 2) { - - // We have filled HiveDecimal.MAX_PRECISION digits and have no more room in our limit precision - // fast decimal. - return false; - } - longWordIndex++; - - // The middle and lowest longwords highest digit number is LONGWORD_DECIMAL_DIGITS. - digitNum = LONGWORD_DECIMAL_DIGITS; - multiplier = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - 1]; - longWord = 0; - } - longWord += digitValue * multiplier; - multiplier /= 10; - digitNum--; - precision++; - if (++index >= end) { - break; - } - } - - // At this point we may have parsed an integer. - - // Try to eat a dot now since it could be the end. We remember if we saw a dot so we can - // do error checking later and detect just a dot. - boolean sawDot = false; - if (index < end && bytes[index] == BYTE_DOT) { - sawDot = true; - index++; - } - - // Try to eat trailing blank padding. - if (trimBlanks && index < end && bytes[index] == BYTE_BLANK) { - index++; - while (index < end && bytes[index] == BYTE_BLANK) { - index++; - } - if (index < end) { - // Junk after trailing blank padding. - return false; - } - // Otherwise, fall through and process the what we saw before possible trailing blanks. - } - - // Any more input? - if (index >= end) { - - // We hit the end after getting optional integer and optional dot and optional blank padding. - - if (!haveInteger) { - return false; - } - - if (precision == 0) { - - // We just had leading zeroes (and possibly a dot and trailing blanks). - // Value is 0. - return true; - } - // Save last longword. - if (longWordIndex == 0) { - fast2 = longWord; - } else if (longWordIndex == 1) { - fast1 = longWord; - } else { - fast0 = longWord; - } - fastResult.fastSignum = (isNegative ? -1 : 1); - fastResult.fastIntegerDigitCount = precision; - fastResult.fastScale = 0; - final int scaleDown = HiveDecimal.MAX_PRECISION - precision; - if (scaleDown > 0) { - doFastScaleDown(fast0, fast1, fast2, scaleDown, fastResult); - } else { - fastResult.fast0 = fast0; - fastResult.fast1 = fast1; - fastResult.fast2 = fast2; - } - return true; - } - - // We have more input but did we start with something valid? - if (!haveInteger && !sawDot) { - - // Must have one of those at this point. - return false; - } - - int integerDigitCount = precision; - - int nonTrailingZeroScale = 0; - boolean roundingNecessary = false; - if (sawDot) { - - // Parse fraction portion. - - while (true) { - work = bytes[index]; - if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) { - if (!haveInteger) { - - // Naked dot. - return false; - } - break; - } - digitValue = work - BYTE_DIGIT_ZERO; - if (digitNum == 0) { - - // Fraction digit parsing move to next lower longword. - - // Save previous longword. - if (longWordIndex == 0) { - fast2 = longWord; - } else if (longWordIndex == 1) { - fast1 = longWord; - } else if (longWordIndex == 2) { - - // We have filled HiveDecimal.MAX_PRECISION digits and have no more room in our limit precision - // fast decimal. However, since we are processing fractional digits, we do rounding. - // away. - if (digitValue >= 5) { - roundingNecessary = true; - } - - // Scan through any remaining digits... - while (++index < end) { - work = bytes[index]; - if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) { - break; - } - } - break; - } - longWordIndex++; - digitNum = LONGWORD_DECIMAL_DIGITS; - multiplier = powerOfTenTable[digitNum - 1]; - longWord = 0; - } - longWord += digitValue * multiplier; - multiplier /= 10; - digitNum--; - precision++; - if (digitValue != 0) { - nonTrailingZeroScale = precision - integerDigitCount; - } - if (++index >= end) { - break; - } - } - } - - boolean haveExponent = false; - if (index < end && - (bytes[index] == BYTE_EXPONENT_UPPER || bytes[index] == BYTE_EXPONENT_LOWER)) { - haveExponent = true; - index++; - if (index >= end) { - // More required. - return false; - } - } - - // At this point we have a number. Save it in fastResult. Round it. If we have an exponent, - // we will do a power 10 operation on fastResult. - - // Save last longword. - if (longWordIndex == 0) { - fast2 = longWord; - } else if (longWordIndex == 1) { - fast1 = longWord; - } else { - fast0 = longWord; - } - - int trailingZeroesScale = precision - integerDigitCount; - if (integerDigitCount == 0 && nonTrailingZeroScale == 0) { - // Zero(es). - } else { - fastResult.fastSignum = (isNegative ? -1 : 1); - fastResult.fastIntegerDigitCount = integerDigitCount; - fastResult.fastScale = nonTrailingZeroScale; - final int trailingZeroCount = trailingZeroesScale - fastResult.fastScale; - final int scaleDown = HiveDecimal.MAX_PRECISION - precision + trailingZeroCount; - if (scaleDown > 0) { - doFastScaleDown(fast0, fast1, fast2, scaleDown, fastResult); - } else { - fastResult.fast0 = fast0; - fastResult.fast1 = fast1; - fastResult.fast2 = fast2; - } - } - - if (roundingNecessary) { - - if (fastResult.fastSignum == 0) { - fastResult.fastSignum = (isNegative ? -1 : 1); - fastResult.fast0 = 1; - fastResult.fastIntegerDigitCount = 0; - fastResult.fastScale = HiveDecimal.MAX_SCALE; - } else { - if (!fastAdd( - fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, - fastResult.fastIntegerDigitCount, fastResult.fastScale, - fastResult.fastSignum, 1, 0, 0, 0, trailingZeroesScale, - fastResult)) { - return false; - } - } - } - - if (!haveExponent) { - - // Try to eat trailing blank padding. - if (trimBlanks && index < end && bytes[index] == BYTE_BLANK) { - index++; - while (index < end && bytes[index] == BYTE_BLANK) { - index++; - } - } - if (index < end) { - // Junk after trailing blank padding. - return false; - } - return true; - } - - // At this point, we have seen the exponent letter E or e and have decimal information as: - // isNegative, precision, integerDigitCount, nonTrailingZeroScale, and - // fast0, fast1, fast2. - // - // After we determine the exponent, we will do appropriate scaling and fill in fastResult. - - boolean isExponentNegative = false; - if (bytes[index] == BYTE_MINUS) { - isExponentNegative = true; - if (++index >= end) { - return false; - } - } else if (bytes[index] == BYTE_PLUS) { - if (++index >= end) { - return false; - } - } - - long exponent = 0; - multiplier = 1; - while (true) { - work = bytes[index]; - if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) { - break; - } - if (multiplier > 10) { - // Power of ten way beyond our precision/scale... - return false; - } - digitValue = work - BYTE_DIGIT_ZERO; - if (digitValue != 0 || exponent != 0) { - exponent = exponent * 10 + digitValue; - multiplier *= 10; - } - if (++index >= end) { - break; - } - } - if (isExponentNegative) { - exponent = -exponent; - } - - // Try to eat trailing blank padding. - if (trimBlanks && index < end && bytes[index] == BYTE_BLANK) { - index++; - while (index < end && bytes[index] == BYTE_BLANK) { - index++; - } - } - if (index < end) { - // Junk after exponent. - return false; - } - - - if (integerDigitCount == 0 && nonTrailingZeroScale == 0) { - // Zero(es). - return true; - } - - if (exponent == 0) { - - // No effect since 10^0 = 1. - - } else { - - // We for these input with exponents, we have at this point an intermediate decimal, - // an exponent power, and a result: - // - // intermediate - // input decimal exponent result - // 701E+1 701 scale 0 +1 7010 scale 0 - // 3E+4 3 scale 0 +4 3 scale 0 - // 3.223E+9 3.223 scale 3 +9 3223000000 scale 0 - // 0.009E+10 0.009 scale 4 +10 90000000 scale 0 - // 0.3221E-2 0.3221 scale 4 -2 0.003221 scale 6 - // 0.00223E-20 0.00223 scale 5 -20 0.0000000000000000000000223 scale 25 - // - - if (!fastScaleByPowerOfTen( - fastResult, - (int) exponent, - fastResult)) { - return false; - } - } - - final int trailingZeroCount = - fastTrailingDecimalZeroCount( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - fastResult.fastIntegerDigitCount, fastResult.fastScale); - if (trailingZeroCount > 0) { - doFastScaleDown( - fastResult, - trailingZeroCount, - fastResult); - fastResult.fastScale -= trailingZeroCount; - } - - return true; - } - - /** - * Scans a byte array slice for UNSIGNED RAW DIGITS ONLY in UTF-8 (ASCII) characters - * and forms a decimal from the digits and a sign and scale. - * - * Designed for BinarySortable serialization format that separates the sign and scale - * from the raw digits. - * - * NOTE: The fastSetFromDigitsOnlyBytesAndScale method requires the caller to pass a fastResult - * parameter has been reset for better performance. - * - * @param isNegative is the number negative - * @param bytes the bytes to read from - * @param offset the position to start at - * @param length the number of bytes to read - * @param scale the scale of the number - * @param fastResult an object it into - * @return True if the sign, digits, and scale were successfully converted to a decimal. - */ - public static boolean fastSetFromDigitsOnlyBytesAndScale( - boolean isNegative, byte[] bytes, int offset, int length, int scale, - FastHiveDecimal fastResult) { - - final int bytesLength = bytes.length; - - if (offset < 0 || offset >= bytesLength) { - return false; - } - final int end = offset + length; - if (end <= offset || end > bytesLength) { - return false; - } - - // We start here with at least one byte. - int index = offset; - - // A stripped down version of fastSetFromBytes. - - int precision = 0; - - // We fill starting with highest digit in highest longword (HIGHWORD_DECIMAL_DIGITS) and - // move down. At end will will shift everything down if necessary. - - int longWordIndex = 0; // Where 0 is the highest longword; 1 is middle longword, etc. - - int digitNum = HIGHWORD_DECIMAL_DIGITS; - long multiplier = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1]; - - int digitValue; - long longWord = 0; - - long fast0 = 0; - long fast1 = 0; - long fast2 = 0; - - byte work; - - // Parse digits. - - boolean haveInteger = false; - while (true) { - work = bytes[index]; - if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) { - if (!haveInteger) { - return false; - } - break; - } - haveInteger = true; - if (precision == 0 && work == BYTE_DIGIT_ZERO) { - // Ignore leading zeroes. - if (++index >= end) { - break; - } - continue; - } - digitValue = work - BYTE_DIGIT_ZERO; - if (digitNum == 0) { - - // Integer parsing move to next lower longword. - - // Save previous longword. - if (longWordIndex == 0) { - fast2 = longWord; - } else if (longWordIndex == 1) { - fast1 = longWord; - } else if (longWordIndex == 2) { - - // We have filled HiveDecimal.MAX_PRECISION digits and have no more room in our limit precision - // fast decimal. - return false; - } - longWordIndex++; - - // The middle and lowest longwords highest digit number is LONGWORD_DECIMAL_DIGITS. - digitNum = LONGWORD_DECIMAL_DIGITS; - multiplier = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - 1]; - longWord = 0; - } - longWord += digitValue * multiplier; - multiplier /= 10; - digitNum--; - precision++; - if (++index >= end) { - break; - } - } - - // Just an digits? - if (index < end) { - return false; - } - - if (precision == 0) { - // We just had leading zeroes. - // Value is 0. - return true; - } - - // Save last longword. - if (longWordIndex == 0) { - fast2 = longWord; - } else if (longWordIndex == 1) { - fast1 = longWord; - } else { - fast0 = longWord; - } - fastResult.fastSignum = (isNegative ? -1 : 1); - fastResult.fastIntegerDigitCount = Math.max(0, precision - scale); - fastResult.fastScale = scale; - final int scaleDown = HiveDecimal.MAX_PRECISION - precision; - if (scaleDown > 0) { - doFastScaleDown(fast0, fast1, fast2, scaleDown, fastResult); - } else { - fastResult.fast0 = fast0; - fastResult.fast1 = fast1; - fastResult.fast2 = fast2; - } - return true; - - } - - /** - * Scale down a BigInteger by a power of 10 and round off if necessary using ROUND_HALF_UP. - * @return The scaled and rounded BigInteger. - */ - private static BigInteger doBigIntegerScaleDown(BigInteger unscaledValue, int scaleDown) { - BigInteger[] quotientAndRemainder = unscaledValue.divideAndRemainder(BigInteger.TEN.pow(scaleDown)); - BigInteger quotient = quotientAndRemainder[0]; - BigInteger round = quotientAndRemainder[1].divide(BigInteger.TEN.pow(scaleDown - 1)); - if (round.compareTo(BIG_INTEGER_FIVE) >= 0) { - quotient = quotient.add(BigInteger.ONE); - } - return quotient; - } - - /** - * Create a fast decimal from a BigDecimal. - * - * NOTE: The fastSetFromBigDecimal method requires the caller to pass a fastResult - * parameter has been reset for better performance. - * - * @param bigDecimal the big decimal to copy - * @param allowRounding is rounding allowed? - * @param fastResult an object to reuse - * @return True if the BigDecimal could be converted to our decimal representation. - */ - public static boolean fastSetFromBigDecimal( - BigDecimal bigDecimal, boolean allowRounding, FastHiveDecimal fastResult) { - - // We trim the trailing zero fraction digits so we don't cause unnecessary precision - // overflow later. - if (bigDecimal.signum() == 0) { - if (bigDecimal.scale() != 0) { - - // For some strange reason BigDecimal 0 can have a scale. We do not support that. - bigDecimal = BigDecimal.ZERO; - } - } - - if (!allowRounding) { - if (bigDecimal.signum() != 0) { - BigDecimal bigDecimalStripped = bigDecimal.stripTrailingZeros(); - int stripTrailingZerosScale = bigDecimalStripped.scale(); - // System.out.println("FAST_SET_FROM_BIG_DECIMAL bigDecimal " + bigDecimal); - // System.out.println("FAST_SET_FROM_BIG_DECIMAL bigDecimalStripped " + bigDecimalStripped); - // System.out.println("FAST_SET_FROM_BIG_DECIMAL stripTrailingZerosScale " + stripTrailingZerosScale); - if (stripTrailingZerosScale < 0) { - - // The trailing zeroes extend into the integer part -- we only want to eliminate the - // fractional zero digits. - - bigDecimal = bigDecimal.setScale(0); - } else { - - // Ok, use result with some or all fractional digits stripped. - - bigDecimal = bigDecimalStripped; - } - } - int scale = bigDecimal.scale(); - if (scale < 0 || scale > HiveDecimal.MAX_SCALE) { - return false; - } - // The digits must fit without rounding. - if (!fastSetFromBigInteger(bigDecimal.unscaledValue(), fastResult)) { - return false; - } - if (fastResult.fastSignum != 0) { - fastResult.fastIntegerDigitCount = Math.max(0, fastResult.fastIntegerDigitCount - scale); - fastResult.fastScale = scale; - } - return true; - } - // This method will scale down and round to fit, if necessary. - if (!fastSetFromBigInteger(bigDecimal.unscaledValue(), bigDecimal.scale(), - bigDecimal.precision(), fastResult)) { - return false; - } - return true; - } - - /** - * Scan a String for a decimal number in UTF-8 characters. - * - * NOTE: The fastSetFromString method requires the caller to pass a fastResult - * parameter has been reset for better performance. - * - * @param string the string to parse - * @param trimBlanks should the blanks be trimmed - * @param result an object to reuse - * @return True if the String was successfully converted to a decimal. - */ - public static boolean fastSetFromString( - String string, boolean trimBlanks, FastHiveDecimal result) { - byte[] bytes = string.getBytes(); - return fastSetFromBytes(bytes, 0, bytes.length, trimBlanks, result); - } - - /** - * Creates a scale 0 fast decimal from an int. - * - * NOTE: The fastSetFromString method requires the caller to pass a fastResult - * parameter has been reset for better performance. - * @param intValue the value to set - * @param fastResult an object to reuse - */ - public static void fastSetFromInt(int intValue, FastHiveDecimal fastResult) { - if (intValue == 0) { - // Zero special case. - return; - } - if (intValue > 0) { - fastResult.fastSignum = 1; - } else { - fastResult.fastSignum = -1; - intValue = Math.abs(intValue); - } - // 10 digit int is all in lowest 16 decimal digit longword. - // Since we are creating with scale 0, no fraction digits to zero trim. - fastResult.fast0 = intValue & 0xFFFFFFFFL; - fastResult.fastIntegerDigitCount = - fastLongWordPrecision(fastResult.fast0); - } - - /** - * Creates a scale 0 fast decimal from a long. - * - * NOTE: The fastSetFromLong method requires the caller to pass a fastResult - * parameter has been reset for better performance. - * - * @param longValue the value to set - * @param fastResult an object to reuse - */ - public static void fastSetFromLong( - long longValue, FastHiveDecimal fastResult) { - if (longValue == 0) { - // Zero special case. - return; - } - // Handle minimum integer case that doesn't have abs(). - if (longValue == Long.MIN_VALUE) { - // Split -9,223,372,036,854,775,808 into 16 digit middle and lowest longwords by hand. - fastResult.fastSignum = -1; - fastResult.fast1 = 922L; - fastResult.fast0 = 3372036854775808L; - fastResult.fastIntegerDigitCount = 19; - } else { - if (longValue > 0) { - fastResult.fastSignum = 1; - } else { - fastResult.fastSignum = -1; - longValue = Math.abs(longValue); - } - // Split into 16 digit middle and lowest longwords remainder / division. - fastResult.fast1 = longValue / MULTIPLER_LONGWORD_DECIMAL; - fastResult.fast0 = longValue % MULTIPLER_LONGWORD_DECIMAL; - if (fastResult.fast1 != 0) { - fastResult.fastIntegerDigitCount = - LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast1); - } else { - fastResult.fastIntegerDigitCount = - fastLongWordPrecision(fastResult.fast0); - } - } - return; - } - - /** - * Creates a fast decimal from a long with a specified scale. - * - * NOTE: The fastSetFromLongAndScale method requires the caller to pass a fastResult - * parameter has been reset for better performance. - * - * @param longValue the value to set as a long - * @param scale the scale to use - * @param fastResult an object to reuse - * @return was the conversion successful? - */ - public static boolean fastSetFromLongAndScale( - long longValue, int scale, FastHiveDecimal fastResult) { - - if (scale < 0 || scale > HiveDecimal.MAX_SCALE) { - return false; - } - - fastSetFromLong(longValue, fastResult); - if (scale == 0) { - return true; - } - - if (!fastScaleByPowerOfTen( - fastResult, - -scale, - fastResult)) { - return false; - } - return true; - } - - /** - * Creates fast decimal from a float. - * - * NOTE: The fastSetFromFloat method requires the caller to pass a fastResult - * parameter has been reset for better performance. - * - * @param floatValue the value to set - * @param fastResult an object to reuse - * @return was the conversion successful? - */ - public static boolean fastSetFromFloat( - float floatValue, FastHiveDecimal fastResult) { - - String floatString = Float.toString(floatValue); - return fastSetFromString(floatString, false, fastResult); - - } - - /** - * Creates fast decimal from a double. - * - * NOTE: The fastSetFromDouble method requires the caller to pass a fastResult - * parameter has been reset for better performance. - * - * @param doubleValue the value to set - * @param fastResult an object to reuse - * @return was the conversion successful? - */ - public static boolean fastSetFromDouble( - double doubleValue, FastHiveDecimal fastResult) { - - String doubleString = Double.toString(doubleValue); - return fastSetFromString(doubleString, false, fastResult); - - } - - /** - * Creates a fast decimal from a BigInteger with scale 0. - * - * For efficiency, we assume that fastResult is fastReset. This method does not set the - * fastScale field. - * - * NOTE: The fastSetFromBigInteger method requires the caller to pass a fastResult - * parameter has been reset for better performance. - * - * @param bigInteger the value to set - * @param fastResult an object to reuse - * @return Return true if the BigInteger value fit within HiveDecimal.MAX_PRECISION. Otherwise, - * false for overflow. - */ - public static boolean fastSetFromBigInteger( - BigInteger bigInteger, FastHiveDecimal fastResult) { - - final int signum = bigInteger.signum(); - if (signum == 0) { - // Zero special case. - return true; - } - fastResult.fastSignum = signum; - if (signum == -1) { - bigInteger = bigInteger.negate(); - } - if (bigInteger.compareTo(BIG_INTEGER_LONGWORD_MULTIPLIER) < 0) { - - // Fits in one longword. - fastResult.fast0 = bigInteger.longValue(); - if (fastResult.fast0 == 0) { - fastResult.fastSignum = 0; - } else { - fastResult.fastIntegerDigitCount = fastLongWordPrecision(fastResult.fast0); - } - return true; - } - BigInteger[] quotientAndRemainder = - bigInteger.divideAndRemainder(BIG_INTEGER_LONGWORD_MULTIPLIER); - fastResult.fast0 = quotientAndRemainder[1].longValue(); - BigInteger quotient = quotientAndRemainder[0]; - if (quotient.compareTo(BIG_INTEGER_LONGWORD_MULTIPLIER) < 0) { - - // Fits in two longwords. - fastResult.fast1 = quotient.longValue(); - if (fastResult.fast0 == 0 && fastResult.fast1 == 0) { - // The special case zero logic at the beginning should have caught this. - throw new RuntimeException("Unexpected"); - } else { - fastResult.fastIntegerDigitCount = - LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast1); - } - return true; - } - - // Uses all 3 decimal longs. - quotientAndRemainder = - quotient.divideAndRemainder(BIG_INTEGER_LONGWORD_MULTIPLIER); - fastResult.fast1 = quotientAndRemainder[1].longValue(); - quotient = quotientAndRemainder[0]; - if (quotient.compareTo(BIG_INTEGER_HIGHWORD_MULTIPLIER) >= 0) { - // Overflow. - return false; - } - fastResult.fast2 = quotient.longValue(); - if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { - fastResult.fastSignum = 0; - } else { - fastResult.fastIntegerDigitCount = - TWO_X_LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(fastResult.fast2); - } - return true; - } - - /** - * Creates a fast decimal from a BigInteger with a specified scale. - * - * NOTE: The fastSetFromBigInteger method requires the caller to pass a fastResult - * parameter has been reset for better performance. - * - * @param bigInteger the value to set as an integer - * @param scale the scale to use - * @param fastResult an object to reuse - * @return True if the BigInteger and scale were successfully converted to a decimal. - */ - public static boolean fastSetFromBigInteger( - BigInteger bigInteger, int scale, FastHiveDecimal fastResult) { - // Poor performance, because the precision will be calculated by bigInteger.toString() - return fastSetFromBigInteger(bigInteger, scale, -1, fastResult); - } - - /** - * Creates a fast decimal from a BigInteger with a specified scale. - * - * NOTE: The fastSetFromBigInteger method requires the caller to pass a fastResult - * parameter has been reset for better performance. - * - * @param bigInteger the value to set as an integer - * @param scale the scale to use - * @param precision the precision to use - * @param fastResult an object to reuse - * @return True if the BigInteger and scale were successfully converted to a decimal. - */ - public static boolean fastSetFromBigInteger( - BigInteger bigInteger, int scale, int precision, FastHiveDecimal fastResult) { - - if (scale < 0) { - - // Multiply by 10^(-scale) to normalize. We do not use negative scale in our representation. - // - // Example: - // 4.172529E+20 has a negative scale -20 since scale is number of digits below the dot. - // 417252900000000000000 normalized as scale 0. - // - bigInteger = bigInteger.multiply(BIG_INTEGER_TEN.pow(-scale)); - scale = 0; - } - - int signum = bigInteger.signum(); - if (signum == 0) { - // Zero. - return true; - } else if (signum == -1) { - // Normalize to positive. - bigInteger = bigInteger.negate(); - } - - if (precision < 0) { - // A slow way to get the number of decimal digits. - precision = bigInteger.toString().length(); - } - - // System.out.println("FAST_SET_FROM_BIG_INTEGER adjusted bigInteger " + bigInteger + " precision " + precision); - - int integerDigitCount = precision - scale; - // System.out.println("FAST_SET_FROM_BIG_INTEGER integerDigitCount " + integerDigitCount + " scale " + scale); - int maxScale; - if (integerDigitCount >= 0) { - if (integerDigitCount > HiveDecimal.MAX_PRECISION) { - return false; - } - maxScale = HiveDecimal.MAX_SCALE - integerDigitCount; - } else { - maxScale = HiveDecimal.MAX_SCALE; - } - // System.out.println("FAST_SET_FROM_BIG_INTEGER maxScale " + maxScale); - - if (scale > maxScale) { - - // A larger scale is ok -- we will knock off lower digits and round. - - final int trimAwayCount = scale - maxScale; - // System.out.println("FAST_SET_FROM_BIG_INTEGER trimAwayCount " + trimAwayCount); - if (trimAwayCount > 1) { - // First, throw away digits below round digit. - BigInteger bigIntegerThrowAwayBelowRoundDigitDivisor = BIG_INTEGER_TEN.pow(trimAwayCount - 1); - bigInteger = bigInteger.divide(bigIntegerThrowAwayBelowRoundDigitDivisor); - } - // System.out.println("FAST_SET_FROM_BIG_INTEGER with round digit bigInteger " + bigInteger + " length " + bigInteger.toString().length()); - - BigInteger[] quotientAndRemainder = bigInteger.divideAndRemainder(BIG_INTEGER_TEN); - // System.out.println("FAST_SET_FROM_BIG_INTEGER quotientAndRemainder " + Arrays.toString(quotientAndRemainder)); - - BigInteger quotient = quotientAndRemainder[0]; - if (quotientAndRemainder[1].intValue() >= 5) { - if (quotient.equals(BIG_INTEGER_MAX_DECIMAL)) { - - // 38 9's digits. - // System.out.println("FAST_SET_FROM_BIG_INTEGER quotient is BIG_INTEGER_MAX_DECIMAL"); - - if (maxScale == 0) { - // No room above for rounding. - return false; - } - - // System.out.println("FAST_SET_FROM_BIG_INTEGER reached here... scale " + scale + " maxScale " + maxScale); - // Rounding results in 10^N. - bigInteger = BIG_INTEGER_TEN.pow(integerDigitCount); - maxScale = 0; - } else { - - // Round up. - bigInteger = quotient.add(BigInteger.ONE); - } - } else { - - // No rounding. - bigInteger = quotient; - } - scale = maxScale; - } - if (!fastSetFromBigInteger(bigInteger, fastResult)) { - return false; - } - - if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { - fastResult.fastSignum = 0; - } else { - fastResult.fastSignum = signum; - fastResult.fastIntegerDigitCount = Math.max(0, fastResult.fastIntegerDigitCount - scale); - fastResult.fastScale = scale; - - final int trailingZeroCount = - fastTrailingDecimalZeroCount( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - fastResult.fastIntegerDigitCount, scale); - if (trailingZeroCount > 0) { - doFastScaleDown( - fastResult, - trailingZeroCount, - fastResult); - fastResult.fastScale -= trailingZeroCount; - } - } - - return true; - } - - //************************************************************************************************ - // Take Integer or Fractional Portion. - - /** - * Creates fast decimal from the fraction portion of a fast decimal. - * - * NOTE: The fastFractionPortion method requires the caller to pass a fastResult - * parameter has been reset for better performance. - * - * @param fastSignum the sign of the number (1, 0, or -1) - * @param fast0 high bits - * @param fast1 second word bits - * @param fast2 third word bits - * @param fastScale the scale - * @param fastResult an object to reuse - */ - public static void fastFractionPortion( - int fastSignum, long fast0, long fast1, long fast2, - int fastScale, - FastHiveDecimal fastResult) { - - if (fastSignum == 0 || fastScale == 0) { - fastResult.fastReset(); - return; - } - - // Clear integer portion; keep fraction. - - // Adjust all longs using power 10 division/remainder. - long result0; - long result1; - long result2; - if (fastScale < LONGWORD_DECIMAL_DIGITS) { - - // Part of lowest word survives. - - final long clearFactor = powerOfTenTable[fastScale]; - - result0 = fast0 % clearFactor; - result1 = 0; - result2 = 0; - - } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - // Throw away lowest word. - - final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS; - - final long clearFactor = powerOfTenTable[adjustedScaleDown]; - - result0 = fast0; - result1 = fast1 % clearFactor; - result2 = 0; - - } else { - - // Throw away middle and lowest words. - - final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS; - - final long clearFactor = powerOfTenTable[adjustedScaleDown]; - - result0 = fast0; - result1 = fast1; - result2 = fast2 % clearFactor; - - } - if (result0 == 0 && result1 == 0 && result2 == 0) { - fastResult.fastReset(); - } else { - fastResult.fastSet(fastSignum, result0, result1, result2, /* fastIntegerDigitCount */ 0, fastScale); - } - } - - /** - * Creates fast decimal from the integer portion. - * - * NOTE: The fastFractionPortion method requires the caller to pass a fastResult - * parameter has been reset for better performance. - * - * @param fastSignum the sign of the number (1, 0, or -1) - * @param fast0 high bits - * @param fast1 second word bits - * @param fast2 third word bits - * @param fastIntegerDigitCount the number of integer digits - * @param fastScale the scale - * @param fastResult an object to reuse - */ - public static void fastIntegerPortion( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - FastHiveDecimal fastResult) { - - if (fastSignum == 0) { - fastResult.fastReset(); - return; - } - if (fastScale == 0) { - fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - } - - // Scale down no rounding to clear fraction. - fastResult.fastSignum = fastSignum; - doFastScaleDown( - fast0, fast1, fast2, - fastScale, - fastResult); - fastResult.fastIntegerDigitCount = fastIntegerDigitCount; - fastResult.fastScale = 0; - } - - //************************************************************************************************ - // Binary to Decimal Conversion. - - /** - * Convert 3 binary words of N bits each to a fast decimal (scale 0). - * - * The 3 binary words highWord, middleWord, and lowerWord form a large binary value: - * - * highWord * 2^(M+L) + middleWord * 2^L + lowerWord. - * - * Where L is the number of bits in the lower word; M is the number of bits in the middle word. - * We let L and M be different to support the SerializationUtil serialization where the lower - * word is 62 bits and the remaining words are 63 bits... - * - * @param lowerWord the lower internal representation - * @param middleWord the middle internal representation - * @param highWord the high internal representation - * @param middleWordMultiplier 2^L - * @param highWordMultiplier 2^(M+L) - * @param fastResult an object to reuse - * @return True if the conversion of the 3 binary words to decimal was successful. - */ - public static boolean doBinaryToDecimalConversion( - long lowerWord, long middleWord, long highWord, - FastHiveDecimal middleWordMultiplier, - FastHiveDecimal highWordMultiplier, - FastHiveDecimal fastResult) { - - /* - * Challenge: How to do the math to get this raw binary back to our decimal form. - * - * Briefly, for the middle and upper binary words, convert the middle/upper word into a decimal - * long words and then multiply those by the binary word's power of 2. - * - * And, add the multiply results into the result decimal longwords. - * - */ - long result0 = - lowerWord % MULTIPLER_LONGWORD_DECIMAL; - long result1 = - lowerWord / MULTIPLER_LONGWORD_DECIMAL; - long result2 = 0; - - if (middleWord != 0 || highWord != 0) { - - if (highWord == 0) { - - // Form result from lower and middle words. - - if (!fastMultiply5x5HalfWords( - middleWord % MULTIPLER_LONGWORD_DECIMAL, - middleWord / MULTIPLER_LONGWORD_DECIMAL, - 0, - middleWordMultiplier.fast0, middleWordMultiplier.fast1, middleWordMultiplier.fast2, - fastResult)) { - return false; - } - - final long calc0 = - result0 - + fastResult.fast0; - result0 = - calc0 % MULTIPLER_LONGWORD_DECIMAL; - final long calc1 = - calc0 / MULTIPLER_LONGWORD_DECIMAL - + result1 - + fastResult.fast1; - result1 = - calc1 % MULTIPLER_LONGWORD_DECIMAL; - result2 = - calc1 / MULTIPLER_LONGWORD_DECIMAL - + fastResult.fast2; - - } else if (middleWord == 0) { - - // Form result from lower and high words. - - if (!fastMultiply5x5HalfWords( - highWord % MULTIPLER_LONGWORD_DECIMAL, - highWord / MULTIPLER_LONGWORD_DECIMAL, - 0, - highWordMultiplier.fast0, highWordMultiplier.fast1, highWordMultiplier.fast2, - fastResult)) { - return false; - } - - final long calc0 = - result0 - + fastResult.fast0; - result0 = - calc0 % MULTIPLER_LONGWORD_DECIMAL; - final long calc1 = - calc0 / MULTIPLER_LONGWORD_DECIMAL - + result1 - + fastResult.fast1; - result1 = - calc1 % MULTIPLER_LONGWORD_DECIMAL; - result2 = - calc1 / MULTIPLER_LONGWORD_DECIMAL - + fastResult.fast2; - - } else { - - // Form result from lower, middle, and middle words. - - if (!fastMultiply5x5HalfWords( - middleWord % MULTIPLER_LONGWORD_DECIMAL, - middleWord / MULTIPLER_LONGWORD_DECIMAL, - 0, - middleWordMultiplier.fast0, middleWordMultiplier.fast1, middleWordMultiplier.fast2, - fastResult)) { - return false; - } - - long middleResult0 = fastResult.fast0; - long middleResult1 = fastResult.fast1; - long middleResult2 = fastResult.fast2; - - if (!fastMultiply5x5HalfWords( - highWord % MULTIPLER_LONGWORD_DECIMAL, - highWord / MULTIPLER_LONGWORD_DECIMAL, - 0, - highWordMultiplier.fast0, highWordMultiplier.fast1, highWordMultiplier.fast2, - fastResult)) { - return false; - } - - long calc0 = - result0 - + middleResult0 - + fastResult.fast0; - result0 = - calc0 % MULTIPLER_LONGWORD_DECIMAL; - long calc1 = - calc0 / MULTIPLER_LONGWORD_DECIMAL - + result1 - + middleResult1 - + fastResult.fast1; - result1 = - calc1 % MULTIPLER_LONGWORD_DECIMAL; - result2 = - calc1 / MULTIPLER_LONGWORD_DECIMAL - + middleResult2 - + fastResult.fast2; - } - } - - // Let caller set negative sign if necessary. - if (result2 != 0) { - fastResult.fastIntegerDigitCount = TWO_X_LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(result2); - fastResult.fastSignum = 1; - } else if (result1 != 0) { - fastResult.fastIntegerDigitCount = LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(result1); - fastResult.fastSignum = 1; - } else if (result0 != 0) { - fastResult.fastIntegerDigitCount = fastHighWordPrecision(result0); - fastResult.fastSignum = 1; - } else { - fastResult.fastIntegerDigitCount = 0; - fastResult.fastSignum = 0; - } - - fastResult.fast0 = result0; - fastResult.fast1 = result1; - fastResult.fast2 = result2; - - return true; - } - - //************************************************************************************************ - // Decimal to Binary Conversion. - - /** - * A helper method that produces a single binary word remainder from a fast decimal (and - * quotient). - * - * The fast decimal is longwords of 16 digits each and we need binary words of 2^N. Since - * we are in decimal form, we have do work to get to convert to binary form. - * - * We effectively need to produce on big binary value (i.e. greater than 64 bits since - * HiveDecimal needs 128 bits of binary which Java does not provide primitive support for) - * from the decimal long words and get the lower N binary bit remainder. - * - * We could try and do decimal division by 2^N to get the (integer) quotient, multiply the - * quotient by 2^N decimal, and finally do a decimal subtract that from the original decimal. - * The resulting decimal can be used to easily get the binary remainder. - * - * However, currently, we do not have fast decimal division. - * - * The "trick" we do here is to remember from your Algebra in school than multiplication and - * division are inverses of each other. - * - * So instead of doing decimal division by 2^N we multiply by the inverse: 2^-N. - * - * We produce 1 binary word (remainder) and a decimal quotient for the higher portion. - * - * @param dividendFast0 The input decimal that will produce a - * single binary word remainder and decimal quotient. - * @param dividendFast1 second word - * @param dividendFast2 third word - * @param fastInverseConst the fast decimal inverse of 2^N = 2^-N - * @param quotientIntegerWordNum the word in the inverse multiplication result - * to find the quotient integer decimal portion - * @param quotientIntegerDigitNum the digit in the result to find the quotient - * integer decimal portion - * @param fastMultiplierConst The fast decimal multiplier for converting the - * quotient integer to the larger number to - * subtract from the input decimal to get the - * remainder. - * @param scratchLongs where to store the result remainder word (index 3) and - * result quotient decimal longwords (indices 0 .. 2) - * @return True if the results were produced without overflow. - */ - public static boolean doDecimalToBinaryDivisionRemainder( - long dividendFast0, long dividendFast1, long dividendFast2, - FastHiveDecimal fastInverseConst, - int quotientIntegerWordNum, - int quotientIntegerDigitNum, - FastHiveDecimal fastMultiplierConst, - long[] scratchLongs) { - - // Multiply by inverse (2^-N) to do the 2^N division. - if (!fastMultiply5x6HalfWords( - dividendFast0, dividendFast1, dividendFast2, - fastInverseConst.fast0, fastInverseConst.fast1, fastInverseConst.fast2, - scratchLongs)) { - // Overflow. - return false; - } - - final long divideFactor = powerOfTenTable[quotientIntegerDigitNum]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - quotientIntegerDigitNum]; - - // Extract the integer portion to get the quotient. - long quotientFast0 = - scratchLongs[quotientIntegerWordNum] / divideFactor - + ((scratchLongs[quotientIntegerWordNum + 1] % divideFactor) * multiplyFactor); - long quotientFast1 = - scratchLongs[quotientIntegerWordNum + 1] / divideFactor - + ((scratchLongs[quotientIntegerWordNum + 2] % divideFactor) * multiplyFactor); - long quotientFast2 = - scratchLongs[quotientIntegerWordNum + 2] / divideFactor; - - // Multiply the integer quotient back out so we can subtract it from the original to get - // the remainder. - if (!fastMultiply5x6HalfWords( - quotientFast0, quotientFast1, quotientFast2, - fastMultiplierConst.fast0, fastMultiplierConst.fast1, fastMultiplierConst.fast2, - scratchLongs)) { - return false; - } - - long quotientMultiplied0 = scratchLongs[0]; - long quotientMultiplied1 = scratchLongs[1]; - long quotientMultiplied2 = scratchLongs[2]; - - if (!doSubtractSameScaleNoUnderflow( - dividendFast0, dividendFast1, dividendFast2, - quotientMultiplied0, quotientMultiplied1, quotientMultiplied2, - scratchLongs)) { - // Underflow. - return false; - } - - long remainderBinaryWord = - scratchLongs[1] * MULTIPLER_LONGWORD_DECIMAL - + scratchLongs[0]; - - // Pack the output into the scratch longs. - scratchLongs[0] = quotientFast0; - scratchLongs[1] = quotientFast1; - scratchLongs[2] = quotientFast2; - - scratchLongs[3] = remainderBinaryWord; - - return true; - } - - /** - * Convert a fast decimal into 3 binary words of N bits each. - * - * The 3 binary words will form a large binary value that is the unsigned unscaled decimal value: - * - * highWord * 2^(M+L) + middleWord * 2^L + lowerWord. - * - * Where L is the number of bits in the lower word; M is the number of bits in the middle word. - * We let L and M be different to support the SerializationUtil serialization where the lower - * word is 62 bits and the remaining words are 63 bits... - * - * The fast decimal is longwords of 16 digits each and we need binary words of 2^N. Since - * we are in decimal form, we have do work to get to convert to binary form. - * - * See the comments for doDecimalToBinaryDivisionRemainder for details on the parameters. - * - * The lowerWord is produced by calling doDecimalToBinaryDivisionRemainder. The quotient from - * that is passed to doDecimalToBinaryDivisionRemainder to produce the middleWord. The final - * quotient is used to produce the highWord. - * - * @return True if the 3 binary words were produced without overflow. Overflow is not expected. - */ - private static boolean doDecimalToBinaryConversion( - long fast0, long fast1, long fast2, - FastHiveDecimal fastInverseConst, - int quotientIntegerWordNum, - int quotientIntegerDigitNum, - FastHiveDecimal fastMultiplierConst, - long[] scratchLongs) { - - long lowerBinaryWord; - long middleBinaryWord = 0; - long highBinaryWord = 0; - - if (fastCompareTo( - 1, - fast0, fast1, fast2, 0, - 1, - fastMultiplierConst.fast0, fastMultiplierConst.fast1, fastMultiplierConst.fast2, 0) < 0) { - - // Optimize: whole decimal fits in one binary word. - - lowerBinaryWord = - fast1 * MULTIPLER_LONGWORD_DECIMAL - + fast0; - - } else { - - // Do division/remainder to get lower binary word; quotient will either be middle decimal - // or be both high and middle decimal that requires another division/remainder. - - if (!doDecimalToBinaryDivisionRemainder( - fast0, fast1, fast2, - fastInverseConst, - quotientIntegerWordNum, - quotientIntegerDigitNum, - fastMultiplierConst, - scratchLongs)) { - // Overflow. - return false; - } - - // Unpack the output. - long quotientFast0 = scratchLongs[0]; - long quotientFast1 = scratchLongs[1]; - long quotientFast2 = scratchLongs[2]; - - lowerBinaryWord = scratchLongs[3]; - - if (fastCompareTo( - 1, - quotientFast0, quotientFast1, quotientFast2, 0, - 1, - fastMultiplierConst.fast0, fastMultiplierConst.fast1, fastMultiplierConst.fast2, 0) < 0) { - - // Optimize: whole decimal fits in two binary words. - - middleBinaryWord = - quotientFast1 * MULTIPLER_LONGWORD_DECIMAL - + quotientFast0; - - } else { - if (!doDecimalToBinaryDivisionRemainder( - quotientFast0, quotientFast1, quotientFast2, - fastInverseConst, - quotientIntegerWordNum, - quotientIntegerDigitNum, - fastMultiplierConst, - scratchLongs)) { - // Overflow. - return false; - } - - highBinaryWord = - scratchLongs[1] * MULTIPLER_LONGWORD_DECIMAL - + scratchLongs[0]; - - middleBinaryWord = scratchLongs[3]; - - } - } - - scratchLongs[0] = lowerBinaryWord; - scratchLongs[1] = middleBinaryWord; - scratchLongs[2] = highBinaryWord; - - return true; - } - - //************************************************************************************************ - // Emulate SerializationUtils Deserialization used by ORC. - - /* - * fastSerializationUtilsRead lower word is 62 bits (the lower bit is used as the sign and is - * removed). So, we need a multiplier 2^62 - * - * 2^62 = - * 4611686018427387904 or - * 4,611,686,018,427,387,904 or - * 461,1686018427387904 (16 digit comma'd) - */ - private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_62 = - new FastHiveDecimal(1, 1686018427387904L, 461L, 0, 19, 0); - - /* - * fastSerializationUtilsRead middle word is 63 bits. So, we need a multiplier 2^63 - * - * 2^63 = - * 9223372036854775808 (Long.MAX_VALUE) or - * 9,223,372,036,854,775,808 or - * 922,3372036854775808 (16 digit comma'd) - */ - private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_63 = - new FastHiveDecimal(1, 3372036854775808L, 922L, 0, 19, 0); - - /* - * fastSerializationUtilsRead high word multiplier: - * - * Multiply by 2^(62 + 63) -- 38 digits or 3 decimal words. - * - * (2^62)*(2^63) = - * 42535295865117307932921825928971026432 or - * (12345678901234567890123456789012345678) - * ( 1 2 3 ) - * 42,535,295,865,117,307,932,921,825,928,971,026,432 or - * 425352,9586511730793292,1825928971026432 (16 digit comma'd) - */ - private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_125 = - new FastHiveDecimal(1, 1825928971026432L, 9586511730793292L, 425352L, 38, 0); - - /* - * Inverse of 2^63 = 2^-63. Please see comments for doDecimalToBinaryDivisionRemainder. - * - * Multiply by 1/2^63 = 1.08420217248550443400745280086994171142578125e-19 to divide by 2^63. - * As 16 digit comma'd 1084202172485,5044340074528008,6994171142578125 - * - * Scale down: 63 = 44 fraction digits + 19 (negative exponent or number of zeros after dot). - * - * 3*16 (48) + 15 --> 63 down shift. - */ - private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_63_INVERSE = - new FastHiveDecimal(1, 6994171142578125L, 5044340074528008L, 1084202172485L, 45, 0); - - /* - * Where in the inverse multiplication result to find the quotient integer decimal portion. - * - * Please see comments for doDecimalToBinaryDivisionRemainder. - */ - private static final int SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_WORD_NUM = 3; - private static final int SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_DIGIT_NUM = 15; - - /** - * Deserialize data written in the format used by the SerializationUtils methods - * readBigInteger/writeBigInteger and create a decimal using the supplied scale. - * - * ORC uses those SerializationUtils methods for its serialization. - * - * A scratch bytes array is necessary to do the binary to decimal conversion for better - * performance. Pass a FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ byte array for - * scratchBytes. - * - * @param inputStream the stream to read from - * @param scale the scale of the number - * @param scratchBytes An array for the binary to decimal conversion for better - * performance. Must have length of - * FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ. - * @param fastResult an object to reuse - * @return The deserialized decimal or null if the conversion failed. - * @throws IOException failures in reading the stream - */ - public static boolean fastSerializationUtilsRead(InputStream inputStream, int scale, - byte[] scratchBytes, - FastHiveDecimal fastResult) throws IOException { - - // Following a suggestion from Gopal, quickly read in the bytes from the stream. - // CONSIDER: Have ORC read the whole input stream into a big byte array with one call to - // the read(byte[] b, int off, int len) method and then let this method read from the big - // byte array. - int readCount = 0; - int input; - do { - input = inputStream.read(); - if (input == -1) { - throw new EOFException("Reading BigInteger past EOF from " + inputStream); - } - scratchBytes[readCount++] = (byte) input; - } while (input >= 0x80); - - /* - * Determine the 3 binary words like what SerializationUtils.readBigInteger does. - */ - - long lowerWord63 = 0; - long middleWord63 = 0; - long highWord63 = 0; - - long work = 0; - int offset = 0; - int readIndex = 0; - long b; - do { - b = scratchBytes[readIndex++]; - work |= (0x7f & b) << (offset % 63); - offset += 7; - // if we've read 63 bits, roll them into the result - if (offset == 63) { - lowerWord63 = work; - work = 0; - } else if (offset % 63 == 0) { - if (offset == 126) { - middleWord63 = work; - } else if (offset == 189) { - highWord63 = work; - } else { - throw new EOFException("Reading more than 3 words of BigInteger"); - } - work = 0; - } - } while (readIndex < readCount); - - if (work != 0) { - if (offset < 63) { - lowerWord63 = work; - } else if (offset < 126) { - middleWord63 = work; - } else if (offset < 189) { - highWord63 =work; - } else { - throw new EOFException("Reading more than 3 words of BigInteger"); - } - } - - // Grab sign bit and shift it away. - boolean isNegative = ((lowerWord63 & 0x1) != 0); - lowerWord63 >>= 1; - - /* - * Use common binary to decimal conversion method we share with fastSetFromBigIntegerBytes. - */ - if (!doBinaryToDecimalConversion( - lowerWord63, middleWord63, highWord63, - FAST_HIVE_DECIMAL_TWO_POWER_62, - FAST_HIVE_DECIMAL_TWO_POWER_125, // 2^(62 + 63) - fastResult)) { - return false; - } - - if (isNegative) { - - // Adjust negative result, again doing what SerializationUtils.readBigInteger does. - if (!doAddSameScaleSameSign( - /* resultSignum */ 1, - fastResult.fast0, fastResult.fast1, fastResult.fast2, - 1, 0, 0, - fastResult)) { - return false; - } - } - - if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { - fastResult.fastSignum = 0; - } else { - fastResult.fastSignum = (isNegative ? -1 : 1); - final int rawPrecision = fastRawPrecision(fastResult); - fastResult.fastIntegerDigitCount = Math.max(0, rawPrecision - scale); - fastResult.fastScale = scale; - - /* - * Just in case we deserialize a decimal with trailing zeroes... - */ - final int resultTrailingZeroCount = - fastTrailingDecimalZeroCount( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - fastResult.fastIntegerDigitCount, fastResult.fastScale); - if (resultTrailingZeroCount > 0) { - doFastScaleDown( - fastResult, - resultTrailingZeroCount, - fastResult); - - fastResult.fastScale -= resultTrailingZeroCount; - } - } - - return true; - } - - //************************************************************************************************ - // Emulate SerializationUtils Serialization used by ORC. - - /** - * Write the value of this decimal just like SerializationUtils.writeBigInteger. It header - * comments are: - * - * Write the arbitrarily sized signed BigInteger in vint format. - * - * Signed integers are encoded using the low bit as the sign bit using zigzag - * encoding. - * - * Each byte uses the low 7 bits for data and the high bit for stop/continue. - * - * Bytes are stored LSB first. - * - * NOTE: - * SerializationUtils.writeBigInteger sometimes pads the result with extra zeroes due to - * BigInteger.bitLength -- we do not emulate that. SerializationUtils.readBigInteger will - * produce the same result for both. - * - * @param outputStream the stream to write to - * @param fastSignum the sign digit (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 of the internal representation - * @param fast2 word 2 of the internal representation - * @param fastIntegerDigitCount unused - * @param fastScale unused - * @param scratchLongs scratch space - * @return True if the decimal was successfully serialized into the output stream. - * @throws IOException for problems in writing - */ - public static boolean fastSerializationUtilsWrite(OutputStream outputStream, - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - long[] scratchLongs) - throws IOException { - - boolean isNegative = (fastSignum == -1); - - /* - * The sign is encoded as the least significant bit. - * - * We need to adjust our decimal before conversion to binary. - * - * Positive: - * Multiply by 2. - * - * Negative: - * Logic in SerializationUtils.writeBigInteger does a negate on the BigInteger. We - * do not have to since FastHiveDecimal stores the numbers unsigned in fast0, fast1, - * and fast2. We do need to subtract one though. - * - * And then multiply by 2 and add in the 1 sign bit. - * - * CONSIDER: This could be combined. - */ - long adjust0; - long adjust1; - long adjust2; - - if (isNegative) { - - // Subtract 1. - long r0 = fast0 - 1; - long r1; - if (r0 < 0) { - adjust0 = r0 + MULTIPLER_LONGWORD_DECIMAL; - r1 = fast1 - 1; - } else { - adjust0 = r0; - r1 = fast1; - } - if (r1 < 0) { - adjust1 = r1 + MULTIPLER_LONGWORD_DECIMAL; - adjust2 = fast2 - 1; - } else { - adjust1 = r1; - adjust2 = fast2; - } - if (adjust2 < 0) { - return false; - } - - // Now multiply by 2 and add 1 sign bit. - r0 = adjust0 * 2 + 1; - adjust0 = - r0 % MULTIPLER_LONGWORD_DECIMAL; - r1 = - adjust1 * 2 - + r0 / MULTIPLER_LONGWORD_DECIMAL; - adjust1 = - r1 % MULTIPLER_LONGWORD_DECIMAL; - adjust2 = - adjust2 * 2 - + r1 / MULTIPLER_LONGWORD_DECIMAL; - - } else { - - // Multiply by 2 to make room for 0 sign bit. - long r0 = fast0 * 2; - adjust0 = - r0 % MULTIPLER_LONGWORD_DECIMAL; - final long r1 = - fast1 * 2 - + r0 / MULTIPLER_LONGWORD_DECIMAL; - adjust1 = - r1 % MULTIPLER_LONGWORD_DECIMAL; - adjust2 = - fast2 * 2 - + r1 / MULTIPLER_LONGWORD_DECIMAL; - - } - - /* - * Use common decimal to binary conversion method we share with fastBigIntegerBytes. - */ - if (!doDecimalToBinaryConversion( - adjust0, adjust1, adjust2, - FAST_HIVE_DECIMAL_TWO_POWER_63_INVERSE, - SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_WORD_NUM, - SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_DIGIT_NUM, - FAST_HIVE_DECIMAL_TWO_POWER_63, - scratchLongs)) { - // Overflow. - return false; - } - - long lowerWord63 = scratchLongs[0]; - long middleWord63 = scratchLongs[1]; - long highWord63 = scratchLongs[2]; - - int wordCount; - if (highWord63 != 0) { - wordCount = 3; - } else if (middleWord63 != 0) { - wordCount = 2; - } else { - wordCount = 1; - } - - // Write out the first 63 bits worth of data. - long lowBits = lowerWord63; - for(int i=0; i < 9; ++i) { - // If this is the last byte, leave the high bit off - if (wordCount == 1 && (lowBits & ~0x7f) == 0) { - outputStream.write((byte) lowBits); - return true; - } else { - outputStream.write((byte) (0x80 | (lowBits & 0x7f))); - lowBits >>>= 7; - } - } - if (wordCount <= 1) { - throw new RuntimeException("Expecting write word count > 1"); - } - - lowBits = middleWord63; - for(int i=0; i < 9; ++i) { - // If this is the last byte, leave the high bit off - if (wordCount == 2 && (lowBits & ~0x7f) == 0) { - outputStream.write((byte) lowBits); - return true; - } else { - outputStream.write((byte) (0x80 | (lowBits & 0x7f))); - lowBits >>>= 7; - } - } - - lowBits = highWord63; - for(int i=0; i < 9; ++i) { - // If this is the last byte, leave the high bit off - if ((lowBits & ~0x7f) == 0) { - outputStream.write((byte) lowBits); - return true; - } else { - outputStream.write((byte) (0x80 | (lowBits & 0x7f))); - lowBits >>>= 7; - } - } - - // Should not get here. - throw new RuntimeException("Unexpected"); - } - - public static long getDecimal64AbsMax(int precision) { - return powerOfTenTable[precision] - 1; - } - - /* - * Deserializes 64-bit decimals up to the maximum 64-bit precision (18 decimal digits). - * - * NOTE: Major assumption: the input decimal64 has already been bounds checked and a least - * has a precision <= DECIMAL64_DECIMAL_DIGITS. We do not bounds check here for better - * performance. - */ - public static void fastDeserialize64( - final long inputDecimal64Long, final int inputScale, - FastHiveDecimal fastResult) { - - long decimal64Long; - if (inputDecimal64Long == 0) { - fastResult.fastReset(); - return; - } else if (inputDecimal64Long > 0) { - fastResult.fastSignum = 1; - decimal64Long = inputDecimal64Long; - } else { - fastResult.fastSignum = -1; - decimal64Long = -inputDecimal64Long; - } - - // Trim trailing zeroes -- but only below the decimal point. - int trimScale = inputScale; - while (trimScale > 0 && decimal64Long % 10 == 0) { - decimal64Long /= 10; - trimScale--; - } - - fastResult.fast2 = 0; - fastResult.fast1 = decimal64Long / MULTIPLER_LONGWORD_DECIMAL; - fastResult.fast0 = decimal64Long % MULTIPLER_LONGWORD_DECIMAL; - - fastResult.fastScale = trimScale; - - fastResult.fastIntegerDigitCount = - Math.max(0, fastRawPrecision(fastResult) - fastResult.fastScale); - } - - /* - * Serializes decimal64 up to the maximum 64-bit precision (18 decimal digits). - * - * NOTE: Major assumption: the fast decimal has already been bounds checked and a least - * has a precision <= DECIMAL64_DECIMAL_DIGITS. We do not bounds check here for better - * performance. - */ - public static long fastSerialize64( - int scale, - int fastSignum, long fast1, long fast0, int fastScale) { - - if (fastSignum == 0) { - return 0; - } else if (fastSignum == 1) { - return (fast1 * MULTIPLER_LONGWORD_DECIMAL + fast0) * powerOfTenTable[scale - fastScale]; - } else { - return -(fast1 * MULTIPLER_LONGWORD_DECIMAL + fast0) * powerOfTenTable[scale - fastScale]; - } - } - - //************************************************************************************************ - // Emulate BigInteger deserialization used by LazyBinary and others. - - /* - * fastSetFromBigIntegerBytes word size we choose is 56 bits to stay below the 64 bit sign bit: - * So, we need a multiplier 2^56 - * - * 2^56 = - * 72057594037927936 or - * 72,057,594,037,927,936 or - * 7,2057594037927936 (16 digit comma'd) - */ - private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_56 = - new FastHiveDecimal(1, 2057594037927936L, 7L, 0, 17, 0); - - /* - * fastSetFromBigIntegerBytes high word multiplier is 2^(56 + 56) - * - * (2^56)*(2^56) = - * 5192296858534827628530496329220096 or - * (1234567890123456789012345678901234) - * ( 1 2 3 ) - * 5,192,296,858,534,827,628,530,496,329,220,096 or - * 51,9229685853482762,8530496329220096 (16 digit comma'd) - */ - private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_112 = - new FastHiveDecimal(1, 8530496329220096L, 9229685853482762L, 51L, 34, 0); - - // Multiply by 1/2^56 or 1.387778780781445675529539585113525390625e-17 to divide by 2^56. - // As 16 digit comma'd 13877787,8078144567552953,9585113525390625 - // - // Scale down: 56 = 39 fraction digits + 17 (negative exponent or number of zeros after dot). - // - // 3*16 (48) + 8 --> 56 down shift. - // - private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_56_INVERSE = - new FastHiveDecimal(1, 9585113525390625L, 8078144567552953L, 13877787L, 40, 0); - - /* - * Where in the inverse multiplication result to find the quotient integer decimal portion. - * - * Please see comments for doDecimalToBinaryDivisionRemainder. - */ - private static final int BIG_INTEGER_BYTES_QUOTIENT_INTEGER_WORD_NUM = 3; - private static final int BIG_INTEGER_BYTES_QUOTIENT_INTEGER_DIGIT_NUM = 8; - - private static final int INITIAL_SHIFT = 48; // 56 bits minus 1 byte. - - // Long masks and values. - private static final long LONG_56_BIT_MASK = 0xFFFFFFFFFFFFFFL; - private static final long LONG_TWO_TO_56_POWER = LONG_56_BIT_MASK + 1L; - private static final long LONG_BYTE_MASK = 0xFFL; - private static final long LONG_BYTE_HIGH_BIT_MASK = 0x80L; - - // Byte values. - private static final byte BYTE_ALL_BITS = (byte) 0xFF; - - /** - * Convert bytes in the format used by BigInteger's toByteArray format (and accepted by its - * constructor) into a decimal using the specified scale. - * - * Our bigIntegerBytes methods create bytes in this format, too. - * - * This method is designed for high performance and does not create an actual BigInteger during - * binary to decimal conversion. - * - * @param bytes the bytes to read from - * @param offset the starting position in the bytes array - * @param length the number of bytes to read from the bytes array - * @param scale the scale of the number - * @param fastResult an object to reused - * @return did the conversion succeed? - */ - public static boolean fastSetFromBigIntegerBytesAndScale( - byte[] bytes, int offset, int length, int scale, - FastHiveDecimal fastResult) { - - final int bytesLength = bytes.length; - - if (offset < 0 || offset >= bytesLength) { - return false; - } - final int end = offset + length; - if (end <= offset || end > bytesLength) { - return false; - } - - final int startOffset = offset; - - // Roughly based on BigInteger code. - - boolean isNegative = (bytes[offset] < 0); - if (isNegative) { - - // Find first non-sign (0xff) byte of input. - while (offset < end) { - if (bytes[offset] != -1) { - break; - } - offset++; - } - if (offset > end) { - return false; - } - } else { - - // Strip leading zeroes -- although there shouldn't be any for a decimal. - - while (offset < end && bytes[offset] == 0) { - offset++; - } - if (offset >= end) { - // Zero. - return true; - } - } - - long lowerWord56 = 0; - long middleWord56 = 0; - long highWord56 = 0; - - int reverseIndex = end; - - long work; - int shift; - - final int lowestCount = Math.min(reverseIndex - offset, 7); - shift = 0; - for (int i = 0; i < lowestCount; i++) { - work = bytes[--reverseIndex] & 0xFF; - lowerWord56 |= work << shift; - shift += 8; - } - - if (reverseIndex <= offset) { - if (isNegative) { - lowerWord56 = ~lowerWord56 & ((1L << shift) - 1); - } - } else { - - // Go on to middle word. - - final int middleCount = Math.min(reverseIndex - offset, 7); - shift = 0; - for (int i = 0; i < middleCount; i++) { - work = bytes[--reverseIndex] & 0xFF; - middleWord56 |= work << shift; - shift += 8; - } - if (reverseIndex <= offset) { - if (isNegative) { - lowerWord56 = ~lowerWord56 & LONG_56_BIT_MASK; - middleWord56 = ~middleWord56 & ((1L << shift) - 1); - } - } else { - - // Go on to high word. - - final int highCount = Math.min(reverseIndex - offset, 7); - shift = 0; - for (int i = 0; i < highCount; i++) { - work = bytes[--reverseIndex] & 0xFF; - highWord56 |= work << shift; - shift += 8; - } - if (isNegative) { - // We only need to apply negation to all 3 words when there are 3 words, etc. - lowerWord56 = ~lowerWord56 & LONG_56_BIT_MASK; - middleWord56 = ~middleWord56 & LONG_56_BIT_MASK; - highWord56 = ~highWord56 & ((1L << shift) - 1); - } - } - } - - if (!doBinaryToDecimalConversion( - lowerWord56, middleWord56, highWord56, - FAST_HIVE_DECIMAL_TWO_POWER_56, - FAST_HIVE_DECIMAL_TWO_POWER_112, // 2^(56 + 56) - fastResult)) { - // Overflow. Use slower alternate. - return doAlternateSetFromBigIntegerBytesAndScale( - bytes, startOffset, length, scale, - fastResult); - } - - // System.out.println("fastSetFromBigIntegerBytesAndScale fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2); - if (isNegative) { - if (!doAddSameScaleSameSign( - /* resultSignum */ 1, - fastResult.fast0, fastResult.fast1, fastResult.fast2, - 1, 0, 0, - fastResult)) { - // Overflow. Use slower alternate. - return doAlternateSetFromBigIntegerBytesAndScale( - bytes, startOffset, length, scale, - fastResult); - } - } - - if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { - fastResult.fastSignum = 0; - } else { - fastResult.fastSignum = (isNegative ? -1 : 1); - fastResult.fastScale = scale; - final int rawPrecision = fastRawPrecision(fastResult); - fastResult.fastIntegerDigitCount = Math.max(0, rawPrecision - scale); - - /* - * Just in case we deserialize a decimal with trailing zeroes... - */ - final int resultTrailingZeroCount = - fastTrailingDecimalZeroCount( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - fastResult.fastIntegerDigitCount, fastResult.fastScale); - if (resultTrailingZeroCount > 0) { - doFastScaleDown( - fastResult, - resultTrailingZeroCount, - fastResult); - - fastResult.fastScale -= resultTrailingZeroCount; - } - } - - return true; - } - - /** - * When fastSetFromBigIntegerBytesAndScale can handle the input because it is too large, - * we fall back to this. - */ - private static boolean doAlternateSetFromBigIntegerBytesAndScale( - byte[] bytes, int offset, int length, int scale, - FastHiveDecimal fastResult) { - - byte[] byteArray = Arrays.copyOfRange(bytes, offset, offset + length); - - BigInteger bigInteger = new BigInteger(byteArray); - // System.out.println("doAlternateSetFromBigIntegerBytesAndScale bigInteger " + bigInteger); - BigDecimal bigDecimal = new BigDecimal(bigInteger, scale); - // System.out.println("doAlternateSetFromBigIntegerBytesAndScale bigDecimal " + bigDecimal); - fastResult.fastReset(); - return fastSetFromBigDecimal(bigDecimal, true, fastResult); - } - - //************************************************************************************************ - // Emulate BigInteger serialization used by LazyBinary, Avro, Parquet, and possibly others. - - public static int fastBigIntegerBytes( - final int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int fastSerializeScale, - long[] scratchLongs, byte[] buffer) { - if (fastSerializeScale != -1) { - return - fastBigIntegerBytesScaled( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - fastSerializeScale, - scratchLongs, buffer); - } else { - return - fastBigIntegerBytesUnscaled( - fastSignum, fast0, fast1, fast2, - scratchLongs, buffer); - } - } - - /** - * Return binary representation of this decimal's BigInteger equivalent unscaled value using - * the format that the BigInteger's toByteArray method returns (and the BigInteger constructor - * accepts). - * - * Used by LazyBinary, Avro, and Parquet serialization. - * - * Scratch objects necessary to do the decimal to binary conversion without actually creating a - * BigInteger object are passed for better performance. - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param scratchLongs scratch array of SCRATCH_LONGS_LEN longs - * @param buffer scratch array of SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes - * @return The number of bytes used for the binary result in buffer. Otherwise, 0 if the - * conversion failed. - */ - public static int fastBigIntegerBytesUnscaled( - final int fastSignum, long fast0, long fast1, long fast2, - long[] scratchLongs, byte[] buffer) { - - /* - * Algorithm: - * 1) Convert decimal to three 56-bit words (three is enough for the decimal since we - * represent the decimal with trailing zeroes trimmed). - * 2) Skip leading zeroes in the words. - * 3) Once we find real data (i.e. a non-zero byte), add a sign byte to buffer if necessary. - * 4) Add bytes from the (rest of) 56-bit words. - * 5) Return byte count. - */ - - if (fastSignum == 0) { - buffer[0] = 0; - return 1; - } - - boolean isNegative = (fastSignum == -1); - - /* - * Use common conversion method we share with fastSerializationUtilsWrite. - */ - if (!doDecimalToBinaryConversion( - fast0, fast1, fast2, - FAST_HIVE_DECIMAL_TWO_POWER_56_INVERSE, - BIG_INTEGER_BYTES_QUOTIENT_INTEGER_WORD_NUM, - BIG_INTEGER_BYTES_QUOTIENT_INTEGER_DIGIT_NUM, - FAST_HIVE_DECIMAL_TWO_POWER_56, - scratchLongs)) { - // Overflow. This is not expected. - return 0; - } - - int byteIndex = 0; - - long word0 = scratchLongs[0]; - long word1 = scratchLongs[1]; - long word2 = scratchLongs[2]; - - if (!isNegative) { - - // Positive number. - - long longWork = 0; - - int shift = INITIAL_SHIFT; - - if (word2 != 0L) { - - // Skip leading zeroes in word2. - - while (true) { - longWork = (word2 >> shift) & LONG_BYTE_MASK; - if (longWork != 0) { - break; - } - if (shift == 0) { - throw new RuntimeException("Unexpected #1"); - } - shift -= Byte.SIZE; - } - - // Now that we have found real data, emit sign byte if necessary. - if ((longWork & LONG_BYTE_HIGH_BIT_MASK) != 0) { - // Add sign byte since high bit is on. - buffer[byteIndex++] = (byte) 0; - } - - // Emit the rest of word2 - while (true) { - buffer[byteIndex++] = (byte) longWork; - if (shift == 0) { - break; - } - shift -= Byte.SIZE; - longWork = (word2 >> shift) & LONG_BYTE_MASK; - } - - shift = INITIAL_SHIFT; - } - - if (byteIndex == 0 && word1 == 0L) { - - // Skip word1, also. - - } else { - - if (byteIndex == 0) { - - // Skip leading zeroes in word1. - - while (true) { - longWork = (word1 >> shift) & LONG_BYTE_MASK; - if (longWork != 0) { - break; - } - if (shift == 0) { - throw new RuntimeException("Unexpected #2"); - } - shift -= Byte.SIZE; - } - - // Now that we have found real data, emit sign byte if necessary. - if ((longWork & LONG_BYTE_HIGH_BIT_MASK) != 0) { - // Add sign byte since high bit is on. - buffer[byteIndex++] = (byte) 0; - } - - } else { - longWork = (word1 >> shift) & LONG_BYTE_MASK; - } - - // Emit the rest of word1 - - while (true) { - buffer[byteIndex++] = (byte) longWork; - if (shift == 0) { - break; - } - shift -= Byte.SIZE; - longWork = (word1 >> shift) & LONG_BYTE_MASK; - } - - shift = INITIAL_SHIFT; - } - - if (byteIndex == 0) { - - // Skip leading zeroes in word0. - - while (true) { - longWork = (word0 >> shift) & LONG_BYTE_MASK; - if (longWork != 0) { - break; - } - if (shift == 0) { - - // All zeroes -- we should have handled this earlier. - throw new RuntimeException("Unexpected #3"); - } - shift -= Byte.SIZE; - } - - // Now that we have found real data, emit sign byte if necessary. - if ((longWork & LONG_BYTE_HIGH_BIT_MASK) != 0) { - // Add sign byte since high bit is on. - buffer[byteIndex++] = (byte) 0; - } - - } else { - longWork = (word0 >> shift) & LONG_BYTE_MASK; - } - - // Emit the rest of word0. - while (true) { - buffer[byteIndex++] = (byte) longWork; - if (shift == 0) { - break; - } - shift -= Byte.SIZE; - longWork = (word0 >> shift) & LONG_BYTE_MASK; - } - - } else { - - // Negative number. - - // Subtract 1 for two's compliment adjustment. - word0--; - if (word0 < 0) { - word0 += LONG_TWO_TO_56_POWER; - word1--; - if (word1 < 0) { - word1 += LONG_TWO_TO_56_POWER; - word2--; - if (word2 < 0) { - // Underflow. - return 0; - } - } - } - - long longWork = 0; - - int shift = INITIAL_SHIFT; - - if (word2 != 0L) { - - // Skip leading zeroes in word2. - - while (true) { - longWork = (word2 >> shift) & LONG_BYTE_MASK; - if (longWork != 0) { - break; - } - if (shift == 0) { - throw new RuntimeException("Unexpected #1"); - } - shift -= Byte.SIZE; - } - - // Now that we have found real data, emit sign byte if necessary and do negative fixup. - - longWork = (~longWork & LONG_BYTE_MASK); - if (((longWork) & LONG_BYTE_HIGH_BIT_MASK) == 0) { - // Add sign byte since high bit is off. - buffer[byteIndex++] = BYTE_ALL_BITS; - } - - // Invert words. - word2 = ~word2; - word1 = ~word1; - word0 = ~word0; - - // Emit the rest of word2 - while (true) { - buffer[byteIndex++] = (byte) longWork; - if (shift == 0) { - break; - } - shift -= Byte.SIZE; - longWork = (word2 >> shift) & LONG_BYTE_MASK; - } - - shift = INITIAL_SHIFT; - } - - if (byteIndex == 0 && word1 == 0L) { - - // Skip word1, also. - - } else { - - if (byteIndex == 0) { - - // Skip leading zeroes in word1. - - while (true) { - longWork = (word1 >> shift) & LONG_BYTE_MASK; - if (longWork != 0) { - break; - } - if (shift == 0) { - throw new RuntimeException("Unexpected #2"); - } - shift -= Byte.SIZE; - } - - // Now that we have found real data, emit sign byte if necessary and do negative fixup. - - longWork = (~longWork & LONG_BYTE_MASK); - if ((longWork & LONG_BYTE_HIGH_BIT_MASK) == 0) { - // Add sign byte since high bit is off. - buffer[byteIndex++] = BYTE_ALL_BITS; - } - - // Invert words. - word1 = ~word1; - word0 = ~word0; - - } else { - longWork = (word1 >> shift) & LONG_BYTE_MASK; - } - - // Emit the rest of word1 - - while (true) { - buffer[byteIndex++] = (byte) longWork; - if (shift == 0) { - break; - } - shift -= Byte.SIZE; - longWork = (word1 >> shift) & LONG_BYTE_MASK; - } - - shift = INITIAL_SHIFT; - } - - if (byteIndex == 0) { - - // Skip leading zeroes in word0. - - while (true) { - longWork = (word0 >> shift) & LONG_BYTE_MASK; - if (longWork != 0) { - break; - } - if (shift == 0) { - - // All zeroes. - - // -1 special case. Unsigned magnitude 1 - two's compliment adjustment 1 = 0. - buffer[0] = BYTE_ALL_BITS; - return 1; - } - shift -= Byte.SIZE; - } - - // Now that we have found real data, emit sign byte if necessary and do negative fixup. - - longWork = (~longWork & LONG_BYTE_MASK); - if ((longWork & LONG_BYTE_HIGH_BIT_MASK) == 0) { - // Add sign byte since high bit is off. - buffer[byteIndex++] = BYTE_ALL_BITS; - } - - // Invert words. - word0 = ~word0; - - } else { - longWork = (word0 >> shift) & LONG_BYTE_MASK; - } - - // Emit the rest of word0. - while (true) { - buffer[byteIndex++] = (byte) longWork; - if (shift == 0) { - break; - } - shift -= Byte.SIZE; - longWork = (word0 >> shift) & LONG_BYTE_MASK; - } - } - - return byteIndex; - } - - /** - * Convert decimal to BigInteger binary bytes with a serialize scale, similar to the formatScale - * for toFormatString. It adds trailing zeroes when a serializeScale is greater than current - * scale. Or, rounds if scale is less than current scale. - * - * Used by Avro and Parquet serialization. - * - * This emulates the OldHiveDecimal setScale / OldHiveDecimal getInternalStorage() behavior. - * - * @param fastSignum the sign number (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param fastIntegerDigitCount the number of integer digits - * @param fastScale the scale - * @param serializeScale the scale to serialize - * @param scratchLongs a scratch array of longs - * @param buffer the buffer to serialize into - * @return the number of bytes used to serialize the number - */ - public static int fastBigIntegerBytesScaled( - final int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int serializeScale, - long[] scratchLongs, byte[] buffer) { - - // Normally, trailing fractional digits are removed. But to emulate the - // OldHiveDecimal setScale and OldHiveDecimalWritable internalStorage, we need to trailing zeroes - // here. - // - // NOTE: This can cause a decimal that has too many decimal digits (because of trailing zeroes) - // for us to represent. In that case, we punt and convert with a BigInteger alternate - // code. - - if (fastSignum == 0 || serializeScale == fastScale) { - return - fastBigIntegerBytesUnscaled( - fastSignum, fast0, fast1, fast2, - scratchLongs, buffer); - } else if (serializeScale > fastScale) { - - final int scaleUp = serializeScale - fastScale; - final int maxScale = HiveDecimal.MAX_SCALE - fastIntegerDigitCount; - if (serializeScale > maxScale) { - - // We cannot to scaled up decimals that cannot be represented. - // Instead, we use a BigInteger instead. - - BigInteger bigInteger = - fastBigIntegerValueUnscaled( - fastSignum, fast0, fast1, fast2); - - BigInteger bigIntegerScaled = bigInteger.multiply(BIG_INTEGER_TEN.pow(scaleUp)); - byte[] bigIntegerBytesScaled = bigIntegerScaled.toByteArray(); - final int length = bigIntegerBytesScaled.length; - System.arraycopy(bigIntegerBytesScaled, 0, buffer, 0, length); - return length; - } - - FastHiveDecimal fastTemp = new FastHiveDecimal(); - if (!fastScaleUp( - fast0, fast1, fast2, - scaleUp, - fastTemp)) { - throw new RuntimeException("Unexpected"); - } - return - fastBigIntegerBytesUnscaled( - fastSignum, fastTemp.fast0, fastTemp.fast1, fastTemp.fast2, - scratchLongs, buffer); - } else { - - // serializeScale < fastScale. - - FastHiveDecimal fastTemp = new FastHiveDecimal(); - if (!fastRound( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - serializeScale, BigDecimal.ROUND_HALF_UP, - fastTemp)) { - return 0; - } - return - fastBigIntegerBytesUnscaled( - fastSignum, fastTemp.fast0, fastTemp.fast1, fastTemp.fast2, - scratchLongs, buffer); - } - } - - //************************************************************************************************ - // Decimal to Integer conversion. - - private static final int MAX_BYTE_DIGITS = 3; - private static final FastHiveDecimal FASTHIVEDECIMAL_MIN_BYTE_VALUE_MINUS_ONE = - new FastHiveDecimal((long) Byte.MIN_VALUE - 1L); - private static final FastHiveDecimal FASTHIVEDECIMAL_MAX_BYTE_VALUE_PLUS_ONE = - new FastHiveDecimal((long) Byte.MAX_VALUE + 1L); - - private static final int MAX_SHORT_DIGITS = 5; - private static final FastHiveDecimal FASTHIVEDECIMAL_MIN_SHORT_VALUE_MINUS_ONE = - new FastHiveDecimal((long) Short.MIN_VALUE - 1L); - private static final FastHiveDecimal FASTHIVEDECIMAL_MAX_SHORT_VALUE_PLUS_ONE = - new FastHiveDecimal((long) Short.MAX_VALUE + 1L); - - private static final int MAX_INT_DIGITS = 10; - private static final FastHiveDecimal FASTHIVEDECIMAL_MIN_INT_VALUE_MINUS_ONE = - new FastHiveDecimal((long) Integer.MIN_VALUE - 1L); - private static final FastHiveDecimal FASTHIVEDECIMAL_MAX_INT_VALUE_PLUS_ONE = - new FastHiveDecimal((long) Integer.MAX_VALUE + 1L); - - private static final FastHiveDecimal FASTHIVEDECIMAL_MIN_LONG_VALUE = - new FastHiveDecimal(Long.MIN_VALUE); - private static final FastHiveDecimal FASTHIVEDECIMAL_MAX_LONG_VALUE = - new FastHiveDecimal(Long.MAX_VALUE); - private static final int MAX_LONG_DIGITS = - FASTHIVEDECIMAL_MAX_LONG_VALUE.fastIntegerDigitCount; - private static final FastHiveDecimal FASTHIVEDECIMAL_MIN_LONG_VALUE_MINUS_ONE = - new FastHiveDecimal("-9223372036854775809"); - private static final FastHiveDecimal FASTHIVEDECIMAL_MAX_LONG_VALUE_PLUS_ONE = - new FastHiveDecimal("9223372036854775808"); - - private static final BigInteger BIG_INTEGER_UNSIGNED_BYTE_MAX_VALUE = BIG_INTEGER_TWO.pow(Byte.SIZE).subtract(BigInteger.ONE); - private static final BigInteger BIG_INTEGER_UNSIGNED_SHORT_MAX_VALUE = BIG_INTEGER_TWO.pow(Short.SIZE).subtract(BigInteger.ONE); - private static final BigInteger BIG_INTEGER_UNSIGNED_INT_MAX_VALUE = BIG_INTEGER_TWO.pow(Integer.SIZE).subtract(BigInteger.ONE); - private static final BigInteger BIG_INTEGER_UNSIGNED_LONG_MAX_VALUE = BIG_INTEGER_TWO.pow(Long.SIZE).subtract(BigInteger.ONE); - - /** - * Is the decimal value a byte? Range -128 to 127. - * Byte.MIN_VALUE Byte.MAX_VALUE - * - * Emulates testing for no value corruption: - * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().byteValue())) - * - * NOTE: Fractional digits are ignored in the test since fastByteValueClip() will - * remove them (round down). - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param fastIntegerDigitCount the number of integer digits - * @param fastScale the scale of the number - * @return True when fastByteValueClip() will return a correct byte. - */ - public static boolean fastIsByte( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - - if (fastIntegerDigitCount < MAX_BYTE_DIGITS) { - - // Definitely a byte; most bytes fall here - return true; - - } else if (fastIntegerDigitCount > MAX_BYTE_DIGITS) { - - // Definitely not a byte. - return false; - - } else if (fastScale == 0) { - if (fast1 != 0 || fast2 != 0) { - return false; - } - if (fastSignum == 1) { - return (fast0 <= Byte.MAX_VALUE); - } else { - return (-fast0 >= Byte.MIN_VALUE); - } - } else { - - // We need to work a little harder for our comparison. Note we round down for - // integer conversion so anything below the next min/max will work. - - if (fastSignum == 1) { - return - (fastCompareTo( - fastSignum, fast0, fast1, fast2, fastScale, - FASTHIVEDECIMAL_MAX_BYTE_VALUE_PLUS_ONE) < 0); - } else { - return - (fastCompareTo( - fastSignum, fast0, fast1, fast2, fastScale, - FASTHIVEDECIMAL_MIN_BYTE_VALUE_MINUS_ONE) > 0); - } - } - } - - // We use "Clip" in the name because this method will return a corrupted value when - // fastIsByte returns false. - public static byte fastByteValueClip( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - - if (fastScale == 0) { - if (fast1 == 0 && fast2 == 0) { - if (fastSignum == 1) { - if (fast0 <= Byte.MAX_VALUE) { - return (byte) fast0; - } - } else { - if (-fast0 >= Byte.MIN_VALUE) { - return (byte) -fast0; - }; - } - } - // SLOW: Do remainder with BigInteger. - BigInteger bigInteger = - fastBigIntegerValueUnscaled( - fastSignum, fast0, fast1, fast2); - return bigInteger.remainder(BIG_INTEGER_UNSIGNED_BYTE_MAX_VALUE).byteValue(); - } else { - - // Adjust all longs using power 10 division/remainder. - long result0; - long result1; - long result2; - if (fastScale < LONGWORD_DECIMAL_DIGITS) { - - // Part of lowest word survives. - - final long divideFactor = powerOfTenTable[fastScale]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale]; - - result0 = - fast0 / divideFactor - + ((fast1 % divideFactor) * multiplyFactor); - result1 = - fast1 / divideFactor - + ((fast2 % divideFactor) * multiplyFactor); - result2 = - fast2 / divideFactor; - - } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - // Throw away lowest word. - - final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS; - - final long divideFactor = powerOfTenTable[adjustedScaleDown]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; - - result0 = - fast1 / divideFactor - + ((fast2 % divideFactor) * multiplyFactor); - result1 = - fast2 / divideFactor; - result2 = 0; - - } else { - - // Throw away middle and lowest words. - - final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS; - - result0 = - fast2 / powerOfTenTable[adjustedScaleDown]; - result1 = 0; - result2 = 0; - - } - - if (result1 == 0 && result2 == 0) { - if (fastSignum == 1) { - if (result0 <= Byte.MAX_VALUE) { - return (byte) result0; - } - } else { - if (-result0 >= Byte.MIN_VALUE) { - return (byte) -result0; - }; - } - } - // SLOW: Do remainder with BigInteger. - BigInteger bigInteger = - fastBigIntegerValueUnscaled( - fastSignum, result0, result1, result2); - return bigInteger.remainder(BIG_INTEGER_UNSIGNED_BYTE_MAX_VALUE).byteValue(); - } - } - - /** - - * @return True when shortValue() will return a correct short. - */ - - /** - * Is the decimal value a short? Range -32,768 to 32,767. - * Short.MIN_VALUE Short.MAX_VALUE - * - * Emulates testing for no value corruption: - * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().shortValue())) - * - * NOTE: Fractional digits are ignored in the test since fastShortValueClip() will - * remove them (round down). - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param fastIntegerDigitCount the number of integer digits - * @param fastScale the scale of the number - * @return True when fastShortValueClip() will return a correct short. - */ - public static boolean fastIsShort( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - - if (fastIntegerDigitCount < MAX_SHORT_DIGITS) { - - // Definitely a short; most shorts fall here - return true; - - } else if (fastIntegerDigitCount > MAX_SHORT_DIGITS) { - - // Definitely not a short. - return false; - - } else if (fastScale == 0) { - if (fast1 != 0 || fast2 != 0) { - return false; - } - if (fastSignum == 1) { - return (fast0 <= Short.MAX_VALUE); - } else { - return (-fast0 >= Short.MIN_VALUE); - } - } else { - - // We need to work a little harder for our comparison. Note we round down for - // integer conversion so anything below the next min/max will work. - - if (fastSignum == 1) { - return - (fastCompareTo( - fastSignum, fast0, fast1, fast2, fastScale, - FASTHIVEDECIMAL_MAX_SHORT_VALUE_PLUS_ONE) < 0); - } else { - return - (fastCompareTo( - fastSignum, fast0, fast1, fast2, fastScale, - FASTHIVEDECIMAL_MIN_SHORT_VALUE_MINUS_ONE) > 0); - } - } - } - - // We use "Clip" in the name because this method will return a corrupted value when - // fastIsShort returns false. - public static short fastShortValueClip( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - - if (fastScale == 0) { - if (fast1 == 0 && fast2 == 0) { - if (fastSignum == 1) { - if (fast0 <= Short.MAX_VALUE) { - return (short) fast0; - } - } else { - if (-fast0 >= Short.MIN_VALUE) { - return (short) -fast0; - }; - } - } - // SLOW: Do remainder with BigInteger. - BigInteger bigInteger = - fastBigIntegerValueUnscaled( - fastSignum, fast0, fast1, fast2); - return bigInteger.remainder(BIG_INTEGER_UNSIGNED_SHORT_MAX_VALUE).shortValue(); - } else { - - // Adjust all longs using power 10 division/remainder. - long result0; - long result1; - long result2; - if (fastScale < LONGWORD_DECIMAL_DIGITS) { - - // Part of lowest word survives. - - final long divideFactor = powerOfTenTable[fastScale]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale]; - - result0 = - fast0 / divideFactor - + ((fast1 % divideFactor) * multiplyFactor); - result1 = - fast1 / divideFactor - + ((fast2 % divideFactor) * multiplyFactor); - result2 = - fast2 / divideFactor; - - } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - // Throw away lowest word. - - final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS; - - final long divideFactor = powerOfTenTable[adjustedScaleDown]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; - - result0 = - fast1 / divideFactor - + ((fast2 % divideFactor) * multiplyFactor); - result1 = - fast2 / divideFactor; - result2 = 0; - - } else { - - // Throw away middle and lowest words. - - final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS; - - result0 = - fast2 / powerOfTenTable[adjustedScaleDown]; - result1 = 0; - result2 = 0; - - } - - if (result1 == 0 && result2 == 0) { - if (fastSignum == 1) { - if (result0 <= Short.MAX_VALUE) { - return (short) result0; - } - } else { - if (-result0 >= Short.MIN_VALUE) { - return (short) -result0; - }; - } - } - // SLOW: Do remainder with BigInteger. - BigInteger bigInteger = - fastBigIntegerValueUnscaled( - fastSignum, result0, result1, result2); - return bigInteger.remainder(BIG_INTEGER_UNSIGNED_SHORT_MAX_VALUE).shortValue(); - } - } - - /** - * Is the decimal value a int? Range -2,147,483,648 to 2,147,483,647. - * Integer.MIN_VALUE Integer.MAX_VALUE - * - * Emulates testing for no value corruption: - * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().intValue())) - * - * NOTE: Fractional digits are ignored in the test since fastIntValueClip() will - * remove them (round down). - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param fastIntegerDigitCount the number of integer digits - * @param fastScale the scale of the number - * @return True when fastIntValueClip() will return a correct int. - */ - public static boolean fastIsInt( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - - if (fastIntegerDigitCount < MAX_INT_DIGITS) { - - // Definitely a int; most ints fall here - return true; - - } else if (fastIntegerDigitCount > MAX_INT_DIGITS) { - - // Definitely not an int. - return false; - - } else if (fastScale == 0) { - if (fast1 != 0 || fast2 != 0) { - return false; - } - if (fastSignum == 1) { - return (fast0 <= Integer.MAX_VALUE); - } else { - return (-fast0 >= Integer.MIN_VALUE); - } - } else { - - // We need to work a little harder for our comparison. Note we round down for - // integer conversion so anything below the next min/max will work. - - if (fastSignum == 1) { - return - (fastCompareTo( - fastSignum, fast0, fast1, fast2, fastScale, - FASTHIVEDECIMAL_MAX_INT_VALUE_PLUS_ONE) < 0); - } else { - return - (fastCompareTo( - fastSignum, fast0, fast1, fast2, fastScale, - FASTHIVEDECIMAL_MIN_INT_VALUE_MINUS_ONE) > 0); - } - } - } - - // We use "Clip" in the name because this method will return a corrupted value when - // fastIsInt returns false. - public static int fastIntValueClip( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - - if (fastScale == 0) { - if (fast1 == 0 && fast2 == 0) { - if (fastSignum == 1) { - if (fast0 <= Integer.MAX_VALUE) { - return (int) fast0; - } - } else { - if (-fast0 >= Integer.MIN_VALUE) { - return (int) -fast0; - }; - } - } - // SLOW: Do remainder with BigInteger. - BigInteger bigInteger = - fastBigIntegerValueUnscaled( - fastSignum, fast0, fast1, fast2); - return bigInteger.remainder(BIG_INTEGER_UNSIGNED_INT_MAX_VALUE).intValue(); - } else { - - // Adjust all longs using power 10 division/remainder. - long result0; - long result1; - long result2; - if (fastScale < LONGWORD_DECIMAL_DIGITS) { - - // Part of lowest word survives. - - final long divideFactor = powerOfTenTable[fastScale]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale]; - - result0 = - fast0 / divideFactor - + ((fast1 % divideFactor) * multiplyFactor); - result1 = - fast1 / divideFactor - + ((fast2 % divideFactor) * multiplyFactor); - result2 = - fast2 / divideFactor; - - } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - // Throw away lowest word. - - final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS; - - final long divideFactor = powerOfTenTable[adjustedScaleDown]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; - - result0 = - fast1 / divideFactor - + ((fast2 % divideFactor) * multiplyFactor); - result1 = - fast2 / divideFactor; - result2 = 0; - - } else { - - // Throw away middle and lowest words. - - final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS; - - result0 = - fast2 / powerOfTenTable[adjustedScaleDown]; - result1 = 0; - result2 = 0; - - } - - if (result1 == 0 && result2 == 0) { - if (fastSignum == 1) { - if (result0 <= Integer.MAX_VALUE) { - return (int) result0; - } - } else { - if (-result0 >= Integer.MIN_VALUE) { - return (int) -result0; - }; - } - } - // SLOW: Do remainder with BigInteger. - BigInteger bigInteger = - fastBigIntegerValueUnscaled( - fastSignum, result0, result1, result2); - return bigInteger.remainder(BIG_INTEGER_UNSIGNED_INT_MAX_VALUE).intValue(); - } - } - - /** - * Is the decimal value a long? Range -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807. - * Long.MIN_VALUE Long.MAX_VALUE - * - * Emulates testing for no value corruption: - * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().longValue())) - * - * NOTE: Fractional digits are ignored in the test since fastLongValueClip() will - * remove them (round down). - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param fastIntegerDigitCount the number of integer digits - * @param fastScale the scale of the number - * @return True when fastLongValueClip() will return a correct long. - */ - public static boolean fastIsLong( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - - if (fastIntegerDigitCount < MAX_LONG_DIGITS) { - - // Definitely a long; most longs fall here - return true; - - } else if (fastIntegerDigitCount > MAX_LONG_DIGITS) { - - // Definitely not a long. - return false; - - } else if (fastScale == 0) { - - // From the above checks, we know fast2 is zero. - - if (fastSignum == 1) { - FastHiveDecimal max = FASTHIVEDECIMAL_MAX_LONG_VALUE; - if (fast1 > max.fast1 || (fast1 == max.fast1 && fast0 > max.fast0)) { - return false; - } - return true; - } else { - FastHiveDecimal min = FASTHIVEDECIMAL_MIN_LONG_VALUE; - if (fast1 > min.fast1 || (fast1 == min.fast1 && fast0 > min.fast0)) { - return false; - } - return true; - } - - } else { - - // We need to work a little harder for our comparison. Note we round down for - // integer conversion so anything below the next min/max will work. - - if (fastSignum == 1) { - return - (fastCompareTo( - fastSignum, fast0, fast1, fast2, fastScale, - FASTHIVEDECIMAL_MAX_LONG_VALUE_PLUS_ONE) < 0); - } else { - return - (fastCompareTo( - fastSignum, fast0, fast1, fast2, fastScale, - FASTHIVEDECIMAL_MIN_LONG_VALUE_MINUS_ONE) > 0); - } - } - } - - // We use "Clip" in the name because this method will return a corrupted value when - // fastIsLong returns false. - public static long fastLongValueClip( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - if (fastSignum == 0) { - return 0; - } - - if (fastScale == 0) { - // Do first comparison as unsigned. - if (fastCompareTo( - 1, fast0, fast1, fast2, fastScale, - FASTHIVEDECIMAL_MAX_LONG_VALUE) <= 0) { - if (fastSignum == 1) { - return - fast1 * MULTIPLER_LONGWORD_DECIMAL - + fast0; - } else { - return - -(fast1 * MULTIPLER_LONGWORD_DECIMAL - + fast0); - } - } if (fastEquals( - fastSignum, fast0, fast1, fast2, fastScale, - FASTHIVEDECIMAL_MIN_LONG_VALUE)) { - return Long.MIN_VALUE; - } else { - // SLOW: Do remainder with BigInteger. - BigInteger bigInteger = - fastBigIntegerValueUnscaled( - fastSignum, fast0, fast1, fast2); - return bigInteger.remainder(BIG_INTEGER_UNSIGNED_LONG_MAX_VALUE).longValue(); - } - } else { - - // Adjust all longs using power 10 division/remainder. - long result0; - long result1; - long result2; - if (fastScale < LONGWORD_DECIMAL_DIGITS) { - - // Part of lowest word survives. - - final long divideFactor = powerOfTenTable[fastScale]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale]; - - result0 = - fast0 / divideFactor - + ((fast1 % divideFactor) * multiplyFactor); - result1 = - fast1 / divideFactor - + ((fast2 % divideFactor) * multiplyFactor); - result2 = - fast2 / divideFactor; - - } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - // Throw away lowest word. - - final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS; - - final long divideFactor = powerOfTenTable[adjustedScaleDown]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; - - result0 = - fast1 / divideFactor - + ((fast2 % divideFactor) * multiplyFactor); - result1 = - fast2 / divideFactor; - result2 = 0; - - } else { - - // Throw away middle and lowest words. - - final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS; - - result0 = - fast2 / powerOfTenTable[adjustedScaleDown]; - result1 = 0; - result2 = 0; - - } - - // Do first comparison as UNSIGNED. - if (fastCompareTo( - 1, result0, result1, result2, /* fastScale */ 0, - FASTHIVEDECIMAL_MAX_LONG_VALUE) <= 0) { - if (fastSignum == 1) { - return - result1 * MULTIPLER_LONGWORD_DECIMAL - + result0; - } else { - return - -(result1 * MULTIPLER_LONGWORD_DECIMAL - + result0); - } - } if (fastEquals( - fastSignum, result0, result1, result2, /* fastScale */ 0, - FASTHIVEDECIMAL_MIN_LONG_VALUE)) { - - // SIGNED comparison to Long.MIN_VALUE decimal. - return Long.MIN_VALUE; - } else { - // SLOW: Do remainder with BigInteger. - BigInteger bigInteger = - fastBigIntegerValueUnscaled( - fastSignum, result0, result1, result2); - return bigInteger.remainder(BIG_INTEGER_UNSIGNED_LONG_MAX_VALUE).longValue(); - } - } - } - - //************************************************************************************************ - // Decimal to Non-Integer conversion. - - public static float fastFloatValue( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - if (fastSignum == 0) { - return 0; - } - BigDecimal bigDecimal = fastBigDecimalValue( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale); - return bigDecimal.floatValue(); - } - - public static double fastDoubleValue( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - if (fastSignum == 0) { - return 0; - } - - // CONSIDER: Looked at the possibility of faster decimal to double conversion by using some - // of their lower level logic that extracts the various parts out of a double. - // The difficulty is Java's rounding rules are byzantine. - - BigDecimal bigDecimal = fastBigDecimalValue( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale); - return bigDecimal.doubleValue(); - } - - /** - * Get a BigInteger representing the decimal's digits without a dot. - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param fastIntegerDigitCount the number of integer digits - * @param fastScale the scale of the number - * @param fastSerializationScale the scale to serialize - * @return Returns a signed BigInteger. - */ - public static BigInteger fastBigIntegerValue( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int fastSerializationScale) { - if (fastSerializationScale != -1) { - return - fastBigIntegerValueScaled( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - fastSerializationScale); - } else { - return - fastBigIntegerValueUnscaled( - fastSignum, fast0, fast1, fast2); - } - } - - public static BigInteger fastBigIntegerValueUnscaled( - int fastSignum, long fast0, long fast1, long fast2) { - - if (fastSignum == 0) { - return BigInteger.ZERO; - } - BigInteger result; - if (fast2 == 0) { - if (fast1 == 0) { - result = - BigInteger.valueOf(fast0); - } else { - result = - BigInteger.valueOf(fast0).add( - BigInteger.valueOf(fast1).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER)); - } - } else { - result = - BigInteger.valueOf(fast0).add( - BigInteger.valueOf(fast1).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER)).add( - BigInteger.valueOf(fast2).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER_2X)); - } - - return (fastSignum == 1 ? result : result.negate()); - } - - public static BigInteger fastBigIntegerValueScaled( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int fastSerializationScale) { - - // Use the serialization scale and create a BigInteger with trailing zeroes (or - // round the decimal) if necessary. - // - // Since we are emulating old behavior and recommending the use of HiveDecimal.bigIntegerBytesScaled - // instead just do it the slow way. Get the BigDecimal.setScale value and return the - // BigInteger. - // - BigDecimal bigDecimal = - fastBigDecimalValue( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale); - bigDecimal = bigDecimal.setScale(fastSerializationScale, RoundingMode.HALF_UP); - return bigDecimal.unscaledValue(); - } - - /** - * Return a BigDecimal representing the decimal. The BigDecimal class is able to accurately - * represent the decimal. - * - * NOTE: We are not representing our decimal as BigDecimal now as OldHiveDecimal did, so this - * is now slower. - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param fastIntegerDigitCount the number of integer digits - * @param fastScale the scale of the number - * @return the BigDecimal equivalent - */ - public static BigDecimal fastBigDecimalValue( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - BigInteger unscaledValue = - fastBigIntegerValueUnscaled( - fastSignum, fast0, fast1, fast2); - return new BigDecimal(unscaledValue, fastScale); - } - - //************************************************************************************************ - // Decimal Comparison. - - public static int fastCompareTo( - int leftSignum, long leftFast0, long leftFast1, long leftFast2, - int leftScale, - FastHiveDecimal fastRight) { - - return - fastCompareTo( - leftSignum, leftFast0, leftFast1, leftFast2, - leftScale, - fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, - fastRight.fastScale); - } - - private static int doCompareToSameScale( - int signum, - long leftFast0, long leftFast1, long leftFast2, - long rightFast0, long rightFast1, long rightFast2) { - - if (leftFast0 == rightFast0 && leftFast1 == rightFast1 && leftFast2 == rightFast2) { - return 0; - } - if (leftFast2 < rightFast2) { - return -signum; - } else if (leftFast2 > rightFast2) { - return signum; - } - if (leftFast1 < rightFast1) { - return -signum; - } else if (leftFast1 > rightFast1){ - return signum; - } - return (leftFast0 < rightFast0 ? -signum : signum); - } - - public static int fastCompareTo( - int leftSignum, long leftFast0, long leftFast1, long leftFast2, - int leftScale, - int rightSignum, long rightFast0, long rightFast1, long rightFast2, - int rightScale) { - - if (leftSignum == 0 && rightSignum == 0) { - return 0; - } - - // Optimization copied from BigDecimal. - int signDiff = leftSignum - rightSignum; - if (signDiff != 0) { - return (signDiff > 0 ? 1 : -1); - } - - // We are here when the left and right are non-zero and have the same sign. - - if (leftScale == rightScale) { - - return doCompareToSameScale( - leftSignum, - leftFast0, leftFast1, leftFast2, - rightFast0, rightFast1, rightFast2); - - } else { - - // How do we handle different scales? - - // We at least know they are not equal. The one with the larger scale has non-zero digits - // below the other's scale (since the scale does not include trailing zeroes). - - // For comparison purposes, we can scale away those digits. And, we can not scale up since - // that could overflow. - - // Use modified portions of doFastScaleDown code here since we do not want to allocate a - // temporary FastHiveDecimal object. - - long compare0; - long compare1; - long compare2; - int scaleDown; - if (leftScale < rightScale) { - - // Scale down right and compare. - scaleDown = rightScale - leftScale; - - // Adjust all longs using power 10 division/remainder. - - if (scaleDown < LONGWORD_DECIMAL_DIGITS) { - // Part of lowest word survives. - - final long divideFactor = powerOfTenTable[scaleDown]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; - - compare0 = - rightFast0 / divideFactor - + ((rightFast1 % divideFactor) * multiplyFactor); - compare1 = - rightFast1 / divideFactor - + ((rightFast2 % divideFactor) * multiplyFactor); - compare2 = - rightFast2 / divideFactor; - } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) { - // Throw away lowest word. - - final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS; - - final long divideFactor = powerOfTenTable[adjustedScaleDown]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; - - compare0 = - rightFast1 / divideFactor - + ((rightFast2 % divideFactor) * multiplyFactor); - compare1 = - rightFast2 / divideFactor; - compare2 = 0; - } else { - // Throw away middle and lowest words. - - final int adjustedScaleDown = scaleDown - TWO_X_LONGWORD_DECIMAL_DIGITS; - - compare0 = - rightFast2 / powerOfTenTable[adjustedScaleDown]; - compare1 = 0; - compare2 = 0; - } - - if (leftFast0 == compare0 && leftFast1 == compare1 && leftFast2 == compare2) { - // Return less than because of right's digits below left's scale. - return -leftSignum; - } - if (leftFast2 < compare2) { - return -leftSignum; - } else if (leftFast2 > compare2) { - return leftSignum; - } - if (leftFast1 < compare1) { - return -leftSignum; - } else if (leftFast1 > compare1){ - return leftSignum; - } - return (leftFast0 < compare0 ? -leftSignum : leftSignum); - - } else { - - // Scale down left and compare. - scaleDown = leftScale - rightScale; - - // Adjust all longs using power 10 division/remainder. - - if (scaleDown < LONGWORD_DECIMAL_DIGITS) { - // Part of lowest word survives. - - final long divideFactor = powerOfTenTable[scaleDown]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; - - compare1 = - leftFast1 / divideFactor - + ((leftFast2 % divideFactor) * multiplyFactor); - compare0 = - leftFast0 / divideFactor - + ((leftFast1 % divideFactor) * multiplyFactor); - compare2 = - leftFast2 / divideFactor; - } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) { - // Throw away lowest word. - - final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS; - - final long divideFactor = powerOfTenTable[adjustedScaleDown]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; - - compare0 = - leftFast1 / divideFactor - + ((leftFast2 % divideFactor) * multiplyFactor); - compare1 = - leftFast2 / divideFactor; - compare2 = 0; - } else { - // Throw away middle and lowest words. - - final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS; - - compare0 = - leftFast2 / powerOfTenTable[adjustedScaleDown]; - compare1 = 0; - compare2 = 0; - } - - if (compare0 == rightFast0 && compare1 == rightFast1 && compare2 == rightFast2) { - // Return greater than because of left's digits below right's scale. - return leftSignum; - } - if (compare2 < rightFast2) { - return -leftSignum; - } else if (compare2 > rightFast2) { - return leftSignum; - } - if (compare1 < rightFast1) { - return -leftSignum; - } else if (compare1 > rightFast1){ - return leftSignum; - } - return (compare0 < rightFast0 ? -leftSignum : leftSignum); - - } - } - } - - public static boolean fastEquals( - int leftSignum, long leftFast0, long leftFast1, long leftFast2, - int leftScale, - FastHiveDecimal fastRight) { - - if (leftSignum == 0) { - return (fastRight.fastSignum == 0); - } - if (leftSignum != fastRight.fastSignum) { - return false; - } - if (leftScale != fastRight.fastScale) { - // We know they are not equal because the one with the larger scale has non-zero digits - // below the other's scale (since the scale does not include trailing zeroes). - return false; - } - return ( - leftFast0 == fastRight.fast0 && leftFast1 == fastRight.fast1 && leftFast2 == fastRight.fast2); - } - - public static boolean fastEquals( - int leftSignum, long leftFast0, long leftFast1, long leftFast2, - int leftScale, - int rightSignum, long rightFast0, long rightFast1, long rightFast2, - int rightScale) { - - if (leftSignum == 0) { - return (rightSignum == 0); - } - if (leftSignum != rightSignum) { - return false; - } - if (leftScale != rightScale) { - // We know they are not equal because the one with the larger scale has non-zero digits - // below the other's scale (since the scale does not include trailing zeroes). - return false; - } - return ( - leftFast0 == rightFast0 && leftFast1 == rightFast1 && leftFast2 == rightFast2); - } - - private static int doCalculateNewFasterHashCode( - int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) { - - long longHashCode; - - long key = fast0; - - // Hash code logic from original calculateLongHashCode - - key = (~key) + (key << 21); // key = (key << 21) - key - 1; - key = key ^ (key >>> 24); - key = (key + (key << 3)) + (key << 8); // key * 265 - key = key ^ (key >>> 14); - key = (key + (key << 2)) + (key << 4); // key * 21 - key = key ^ (key >>> 28); - key = key + (key << 31); - - longHashCode = key; - - key = fast1; - - key = (~key) + (key << 21); // key = (key << 21) - key - 1; - key = key ^ (key >>> 24); - key = (key + (key << 3)) + (key << 8); // key * 265 - key = key ^ (key >>> 14); - key = (key + (key << 2)) + (key << 4); // key * 21 - key = key ^ (key >>> 28); - key = key + (key << 31); - - longHashCode ^= key; - - key = fast2; - - key = (~key) + (key << 21); // key = (key << 21) - key - 1; - key = key ^ (key >>> 24); - key = (key + (key << 3)) + (key << 8); // key * 265 - key = key ^ (key >>> 14); - key = (key + (key << 2)) + (key << 4); // key * 21 - key = key ^ (key >>> 28); - key = key + (key << 31); - - longHashCode ^= key; - - key = fastSignum; - - key = (~key) + (key << 21); // key = (key << 21) - key - 1; - key = key ^ (key >>> 24); - key = (key + (key << 3)) + (key << 8); // key * 265 - key = key ^ (key >>> 14); - key = (key + (key << 2)) + (key << 4); // key * 21 - key = key ^ (key >>> 28); - key = key + (key << 31); - - longHashCode ^= key; - - key = fastIntegerDigitCount; - - key = (~key) + (key << 21); // key = (key << 21) - key - 1; - key = key ^ (key >>> 24); - key = (key + (key << 3)) + (key << 8); // key * 265 - key = key ^ (key >>> 14); - key = (key + (key << 2)) + (key << 4); // key * 21 - key = key ^ (key >>> 28); - key = key + (key << 31); - - longHashCode ^= key; - - key = fastScale; - - key = (~key) + (key << 21); // key = (key << 21) - key - 1; - key = key ^ (key >>> 24); - key = (key + (key << 3)) + (key << 8); // key * 265 - key = key ^ (key >>> 14); - key = (key + (key << 2)) + (key << 4); // key * 21 - key = key ^ (key >>> 28); - key = key + (key << 31); - - longHashCode ^= key; - - return (int) longHashCode; - } - - private static final int ZERO_NEW_FASTER_HASH_CODE = doCalculateNewFasterHashCode(0, 0, 0, 0, 0, 0); - - /** - * Hash code based on (new) decimal representation. - * - * Faster than fastHashCode(). - * - * Used by map join and other Hive internal purposes where performance is important. - * - * IMPORTANT: See comments for fastHashCode(), too. - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param fastIntegerDigitCount the number of integer digits - * @param fastScale the scale of the number - * @return the hash code - */ - public static int fastNewFasterHashCode( - int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) { - if (fastSignum == 0) { - return ZERO_NEW_FASTER_HASH_CODE; - } - int hashCode = doCalculateNewFasterHashCode(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - return hashCode; - } - - /** - * This is the original hash code as returned by OldHiveDecimal. - * - * We need this when the OldHiveDecimal hash code has been exposed and and written or affected - * how data is written. - * - * This method supports compatibility. - * - * Examples: bucketing and the Hive hash() function. - * - * NOTE: It is necessary to create a BigDecimal object and use its hash code, so this method is - * slow. - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param fastIntegerDigitCount the number of integer digits - * @param fastScale the scale of the number - * @return the hash code - */ - public static int fastHashCode( - int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) { - - // OldHiveDecimal returns the hash code of its internal BigDecimal. Our TestHiveDecimal - // verifies the OldHiveDecimal.bigDecimalValue() matches (new) HiveDecimal.bigDecimalValue(). - - BigDecimal bigDecimal = - fastBigDecimalValue( - fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - - return bigDecimal.hashCode(); - } - - //************************************************************************************************ - // Decimal Math. - - public static boolean fastScaleByPowerOfTen( - FastHiveDecimal fastDec, - int power, - FastHiveDecimal fastResult) { - return fastScaleByPowerOfTen( - fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2, - fastDec.fastIntegerDigitCount, fastDec.fastScale, - power, - fastResult); - } - - // NOTE: power can be positive or negative. - // NOTE: e.g. power = 2 is effectively multiply by 10^2 - // NOTE: and power = -3 is multiply by 10^-3 - public static boolean fastScaleByPowerOfTen( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int power, - FastHiveDecimal fastResult) { - - if (fastSignum == 0) { - fastResult.fastReset(); - return true; - } - if (power == 0) { - fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - /* - if (!fastResult.fastIsValid()) { - fastResult.fastRaiseInvalidException(); - } - */ - return true; - } - - final int absPower = Math.abs(power); - - if (power > 0) { - - int integerRoom; - int fractionalRoom; - if (fastIntegerDigitCount > 0) { - - // Is there integer room above? - - integerRoom = HiveDecimal.MAX_PRECISION - fastIntegerDigitCount; - if (integerRoom < power) { - return false; - } - fastResult.fastSignum = fastSignum; - if (fastScale <= power) { - - // All fractional digits become integer digits. - final int scaleUp = power - fastScale; - if (scaleUp > 0) { - if (!fastScaleUp( - fast0, fast1, fast2, - scaleUp, - fastResult)) { - throw new RuntimeException("Unexpected"); - } - } else { - fastResult.fast0 = fast0; - fastResult.fast1 = fast1; - fastResult.fast2 = fast2; - } - fastResult.fastIntegerDigitCount = fastIntegerDigitCount + fastScale + scaleUp; - fastResult.fastScale = 0; - - } else { - - // Only a scale adjustment is needed. - fastResult.fastSet( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount + power, fastScale - power); - } - } else { - - // How much can the fraction be moved up? - - final int rawPrecision = fastRawPrecision(fastSignum, fast0, fast1, fast2); - final int zeroesBelowDot = fastScale - rawPrecision; - - // Our limit is max precision integer digits + "leading" zeros below the dot. - // E.g. 0.00021 has 3 zeroes below the dot. - // - if (power > HiveDecimal.MAX_PRECISION + zeroesBelowDot) { - - // Fractional part powered up too high. - return false; - } - - final int newIntegerDigitCount = Math.max(0, power - zeroesBelowDot); - if (newIntegerDigitCount > rawPrecision) { - - fastResult.fastSignum = fastSignum; - final int scaleUp = newIntegerDigitCount - rawPrecision; - if (!fastScaleUp( - fast0, fast1, fast2, - scaleUp, - fastResult)) { - throw new RuntimeException("Unexpected"); - } - fastResult.fastIntegerDigitCount = newIntegerDigitCount; - fastResult.fastScale = 0; - } else { - final int newScale = Math.max(0, fastScale - power); - fastResult.fastSet(fastSignum, fast0, fast1, fast2, newIntegerDigitCount, newScale); - } - } - - } else if (fastScale + absPower <= HiveDecimal.MAX_SCALE) { - - // Negative power with range -- adjust the scale. - - final int newScale = fastScale + absPower; - final int newIntegerDigitCount = Math.max(0, fastIntegerDigitCount - absPower); - - final int trailingZeroCount = - fastTrailingDecimalZeroCount( - fast0, fast1, fast2, - newIntegerDigitCount, newScale); - if (trailingZeroCount > 0) { - fastResult.fastSignum = fastSignum; - doFastScaleDown( - fast0, fast1, fast2, - trailingZeroCount, fastResult); - fastResult.fastScale = newScale - trailingZeroCount; - fastResult.fastIntegerDigitCount = newIntegerDigitCount; - } else { - fastResult.fastSet(fastSignum, fast0, fast1, fast2, - newIntegerDigitCount, newScale); - } - } else { - - // fastScale + absPower > HiveDecimal.MAX_SCALE - - // Look at getting rid of fractional digits that will now be below HiveDecimal.MAX_SCALE. - - final int scaleDown = fastScale + absPower - HiveDecimal.MAX_SCALE; - - if (scaleDown < HiveDecimal.MAX_SCALE) { - if (!fastRoundFractionalHalfUp( - fastSignum, fast0, fast1, fast2, - scaleDown, - fastResult)) { - // Overflow. - return false; - } - if (fastResult.fastSignum != 0) { - - fastResult.fastScale = HiveDecimal.MAX_SCALE; - fastResult.fastIntegerDigitCount = - Math.max(0, fastRawPrecision(fastResult) - fastResult.fastScale); - - final int trailingZeroCount = - fastTrailingDecimalZeroCount( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - fastResult.fastIntegerDigitCount, fastResult.fastScale); - if (trailingZeroCount > 0) { - doFastScaleDown( - fastResult, - trailingZeroCount, - fastResult); - fastResult.fastScale -= trailingZeroCount; - } - } - } else { - // All precision has been lost -- result is 0. - fastResult.fastReset(); - } - } - - /* - if (!fastResult.fastIsValid()) { - fastResult.fastRaiseInvalidException(); - } - */ - - return true; - } - - //************************************************************************************************ - // Decimal Rounding. - - public static boolean doFastRound( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int roundPower, - int roundingMode, - FastHiveDecimal fastResult) { - - if (fastSignum == 0) { - - // Zero result. - fastResult.fastReset(); - return true; - } else if (fastScale == roundPower) { - - // The roundPower same as scale means all zeroes below round point. - - fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - /* - if (!fastResult.fastIsValid()) { - fastResult.fastRaiseInvalidException(); - } - */ - return true; - } - - if (roundPower > fastScale) { - - // We pretend to add trailing zeroes, EVEN WHEN it would exceed the HiveDecimal.MAX_PRECISION. - - // Copy current value; do not change current scale. - fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - /* - if (!fastResult.fastIsValid()) { - fastResult.fastRaiseInvalidException(); - } - */ - } else if (roundPower < 0) { - - // roundPower < 0 - // - // Negative scale means we start rounding integer digits. - // - // The result will integer result will have at least abs(roundPower) trailing digits. - // - // Examples where the 'r's show the rounding digits: - // - // round(12500, -3) = 13000 // BigDecimal.ROUND_HALF_UP - // rrr - // - // Or, ceiling(12400.8302, -2) = 12500 // BigDecimal.ROUND_CEILING - // rr rrrr - // - // Notice that any fractional digits will be gone in the result. - // - switch (roundingMode) { - case BigDecimal.ROUND_DOWN: - if (!fastRoundIntegerDown( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - roundPower, - fastResult)) { - return false; - } - break; - case BigDecimal.ROUND_UP: - if (!fastRoundIntegerUp( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - roundPower, - fastResult)) { - return false; - } - break; - case BigDecimal.ROUND_FLOOR: - // Round towards negative infinity. - if (fastSignum == 1) { - if (!fastRoundIntegerDown( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - roundPower, - fastResult)) { - return false; - } - } else { - if (!fastRoundIntegerUp( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - roundPower, - fastResult)) { - return false; - } - if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) { - return false; - } - } - break; - case BigDecimal.ROUND_CEILING: - // Round towards positive infinity. - if (fastSignum == 1) { - if (!fastRoundIntegerUp( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - roundPower, - fastResult)) { - return false; - } - if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) { - return false; - } - } else { - if (!fastRoundIntegerDown( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - roundPower, - fastResult)) { - return false; - } - } - break; - case BigDecimal.ROUND_HALF_UP: - if (!fastRoundIntegerHalfUp( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - roundPower, - fastResult)) { - return false; - } - if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) { - return false; - } - break; - case BigDecimal.ROUND_HALF_EVEN: - if (!fastRoundIntegerHalfEven( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - roundPower, - fastResult)) { - return false; - } - if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) { - return false; - } - break; - default: - throw new RuntimeException("Unsupported rounding mode " + roundingMode); - } - - // The fastRoundInteger* methods remove all fractional digits, set fastIntegerDigitCount, and - // set fastScale to 0. - return true; - - } else { - - // roundPower < fastScale - - // Do rounding of fractional digits. - final int scaleDown = fastScale - roundPower; - switch (roundingMode) { - case BigDecimal.ROUND_DOWN: - fastRoundFractionalDown( - fastSignum, fast0, fast1, fast2, - scaleDown, - fastResult); - break; - case BigDecimal.ROUND_UP: - if (!fastRoundFractionalUp( - fastSignum, fast0, fast1, fast2, - scaleDown, - fastResult)) { - return false; - } - break; - case BigDecimal.ROUND_FLOOR: - // Round towards negative infinity. - if (fastSignum == 1) { - fastRoundFractionalDown( - fastSignum, fast0, fast1, fast2, - scaleDown, - fastResult); - } else { - if (!fastRoundFractionalUp( - fastSignum, fast0, fast1, fast2, - scaleDown, - fastResult)) { - return false; - } - } - break; - case BigDecimal.ROUND_CEILING: - // Round towards positive infinity. - if (fastSignum == 1) { - if (!fastRoundFractionalUp( - fastSignum, fast0, fast1, fast2, - scaleDown, - fastResult)) { - return false; - } - } else { - fastRoundFractionalDown( - fastSignum, fast0, fast1, fast2, - scaleDown, - fastResult); - } - break; - case BigDecimal.ROUND_HALF_UP: - if (!fastRoundFractionalHalfUp( - fastSignum, fast0, fast1, fast2, - scaleDown, - fastResult)) { - return false; - } - break; - case BigDecimal.ROUND_HALF_EVEN: - if (!fastRoundFractionalHalfEven( - fastSignum, fast0, fast1, fast2, - scaleDown, - fastResult)) { - return false; - } - break; - default: - throw new RuntimeException("Unsupported rounding mode " + roundingMode); - } - if (fastResult.fastSignum == 0) { - fastResult.fastScale = 0; - /* - if (!fastResult.fastIsValid()) { - fastResult.fastRaiseInvalidException(); - } - */ - } else { - final int rawPrecision = fastRawPrecision(fastResult); - fastResult.fastIntegerDigitCount = Math.max(0, rawPrecision - roundPower); - fastResult.fastScale = roundPower; - - // Trim trailing zeroes and re-adjust scale. - final int trailingZeroCount = - fastTrailingDecimalZeroCount( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - fastResult.fastIntegerDigitCount, fastResult.fastScale); - if (trailingZeroCount > 0) { - doFastScaleDown( - fastResult, - trailingZeroCount, - fastResult); - fastResult.fastScale -= trailingZeroCount; - /* - if (!fastResult.fastIsValid()) { - fastResult.fastRaiseInvalidException(); - } - */ - } - } - } - - return true; - } - - public static boolean fastRound( - FastHiveDecimal fastDec, - int newScale, int roundingMode, - FastHiveDecimal fastResult) { - return fastRound( - fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2, - fastDec.fastIntegerDigitCount, fastDec.fastScale, - newScale, roundingMode, - fastResult); - } - - public static boolean fastRound( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int newScale, int roundingMode, - FastHiveDecimal fastResult) { - return doFastRound( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - newScale, roundingMode, - fastResult); - } - - private static boolean isRoundPortionAllZeroes( - long fast0, long fast1, long fast2, - int roundingPoint) { - - boolean isRoundPortionAllZeroes; - if (roundingPoint < LONGWORD_DECIMAL_DIGITS) { - - // Lowest word gets integer rounding. - - // Factor includes scale. - final long roundPointFactor = powerOfTenTable[roundingPoint]; - - isRoundPortionAllZeroes = (fast0 % roundPointFactor == 0); - - } else if (roundingPoint < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - // Middle word gets integer rounding. - - final int adjustedRoundingPoint = roundingPoint - LONGWORD_DECIMAL_DIGITS; - - if (adjustedRoundingPoint == 0) { - isRoundPortionAllZeroes = (fast0 == 0); - } else { - - // Factor includes scale. - final long roundPointFactor = powerOfTenTable[adjustedRoundingPoint]; - - final long roundPortion = fast1 % roundPointFactor; - isRoundPortionAllZeroes = (fast0 == 0 && roundPortion == 0); - } - - } else { - - // High word gets integer rounding. - - final int adjustedRoundingPoint = roundingPoint - TWO_X_LONGWORD_DECIMAL_DIGITS; - - if (adjustedRoundingPoint == 0) { - isRoundPortionAllZeroes = (fast0 == 0 && fast1 == 0); - } else { - - // Factor includes scale. - final long roundPointFactor = powerOfTenTable[adjustedRoundingPoint]; - - final long roundPortion = fast2 % roundPointFactor; - isRoundPortionAllZeroes = (fast0 == 0 && fast1 == 0 && roundPortion == 0); - } - } - return isRoundPortionAllZeroes; - } - - private static boolean isRoundPortionHalfUp( - long fast0, long fast1, long fast2, - int roundingPoint) { - - boolean isRoundPortionHalfUp; - if (roundingPoint < LONGWORD_DECIMAL_DIGITS) { - - // Lowest word gets integer rounding. - - // Divide down just before round point to get round digit. - final long withRoundDigit = fast0 / powerOfTenTable[roundingPoint - 1]; - final long roundDigit = withRoundDigit % 10; - - isRoundPortionHalfUp = (roundDigit >= 5); - - } else if (roundingPoint < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - // Middle word gets integer rounding. - - final int adjustedRoundingPoint = roundingPoint - LONGWORD_DECIMAL_DIGITS; - - long roundDigit; - if (adjustedRoundingPoint == 0) { - // Grab round digit from lowest word. - roundDigit = fast0 / (MULTIPLER_LONGWORD_DECIMAL / 10); - } else { - // Divide down just before scaleDown to get round digit. - final long withRoundDigit = fast1 / powerOfTenTable[adjustedRoundingPoint - 1]; - roundDigit = withRoundDigit % 10; - } - - isRoundPortionHalfUp = (roundDigit >= 5); - - } else { - - // High word gets integer rounding. - - final int adjustedRoundingPoint = roundingPoint - TWO_X_LONGWORD_DECIMAL_DIGITS; - - long roundDigit; - if (adjustedRoundingPoint == 0) { - // Grab round digit from middle word. - roundDigit = fast1 / (MULTIPLER_LONGWORD_DECIMAL / 10); - } else { - // Divide down just before scaleDown to get round digit. - final long withRoundDigit = fast2 / powerOfTenTable[adjustedRoundingPoint - 1]; - roundDigit = withRoundDigit % 10; - } - - isRoundPortionHalfUp = (roundDigit >= 5); - - } - return isRoundPortionHalfUp; - } - - private static boolean isRoundPortionHalfEven( - long fast0, long fast1, long fast2, - int roundingPoint) { - - boolean isRoundPortionHalfEven; - if (roundingPoint < LONGWORD_DECIMAL_DIGITS) { - - // Lowest word gets integer rounding. - - // Divide down just before scaleDown to get round digit. - final long roundDivisor = powerOfTenTable[roundingPoint - 1]; - final long withRoundDigit = fast0 / roundDivisor; - final long roundDigit = withRoundDigit % 10; - final long fast0Scaled = withRoundDigit / 10; - - if (roundDigit > 5) { - isRoundPortionHalfEven = true; - } else if (roundDigit == 5) { - boolean exactlyOneHalf; - if (roundingPoint - 1 == 0) { - // Fraction below 0.5 is implicitly 0. - exactlyOneHalf = true; - } else { - exactlyOneHalf = (fast0 % roundDivisor == 0); - } - - // When fraction is exactly 0.5 and lowest new digit is odd, go towards even. - if (exactlyOneHalf) { - isRoundPortionHalfEven = (fast0Scaled % 2 == 1); - } else { - isRoundPortionHalfEven = true; - } - } else { - isRoundPortionHalfEven = false; - } - - } else if (roundingPoint < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - // Middle word gets integer rounding. - - final int adjustedRoundingPoint = roundingPoint - LONGWORD_DECIMAL_DIGITS; - - long roundDigit; - long fast1Scaled; - if (adjustedRoundingPoint == 0) { - // Grab round digit from lowest word. - final long roundDivisor = MULTIPLER_LONGWORD_DECIMAL / 10; - roundDigit = fast0 / roundDivisor; - fast1Scaled = fast1; - if (roundDigit > 5) { - isRoundPortionHalfEven = true; - } else if (roundDigit == 5) { - boolean exactlyOneHalf = (fast0 % roundDivisor == 0); - - // When fraction is exactly 0.5 and lowest new digit is odd, go towards even. - if (exactlyOneHalf) { - isRoundPortionHalfEven = (fast1Scaled % 2 == 1); - } else { - isRoundPortionHalfEven = true; - } - } else { - isRoundPortionHalfEven = false; - } - } else { - // Divide down just before scaleDown to get round digit. - final long roundDivisor = powerOfTenTable[adjustedRoundingPoint - 1]; - final long withRoundDigit = fast1 / roundDivisor; - roundDigit = withRoundDigit % 10; - fast1Scaled = withRoundDigit / 10; - if (roundDigit > 5) { - isRoundPortionHalfEven = true; - } else if (roundDigit == 5) { - boolean exactlyOneHalf; - if (adjustedRoundingPoint - 1 == 0) { - // Just examine the lower word. - exactlyOneHalf = (fast0 == 0); - } else { - exactlyOneHalf = (fast0 == 0 && fast1 % roundDivisor == 0); - } - - // When fraction is exactly 0.5 and lowest new digit is odd, go towards even. - if (exactlyOneHalf) { - isRoundPortionHalfEven = (fast1Scaled % 2 == 1); - } else { - isRoundPortionHalfEven = true; - } - } else { - isRoundPortionHalfEven = false; - } - } - - } else { - - // High word gets integer rounding. - - final int adjustedRoundingPoint = roundingPoint - TWO_X_LONGWORD_DECIMAL_DIGITS; - - long roundDigit; - long fast2Scaled; - if (adjustedRoundingPoint == 0) { - // Grab round digit from middle word. - final long roundDivisor = MULTIPLER_LONGWORD_DECIMAL / 10; - roundDigit = fast1 / roundDivisor; - fast2Scaled = fast2; - if (roundDigit > 5) { - isRoundPortionHalfEven = true; - } else if (roundDigit == 5) { - boolean exactlyOneHalf = (fast1 % roundDivisor == 0 && fast0 == 0); - - // When fraction is exactly 0.5 and lowest new digit is odd, go towards even. - if (exactlyOneHalf) { - isRoundPortionHalfEven = (fast2Scaled % 2 == 1); - } else { - isRoundPortionHalfEven = true; - } - } else { - isRoundPortionHalfEven = false; - } - } else { - // Divide down just before scaleDown to get round digit. - final long roundDivisor = powerOfTenTable[adjustedRoundingPoint - 1]; - final long withRoundDigit = fast2 / roundDivisor; - roundDigit = withRoundDigit % 10; - fast2Scaled = withRoundDigit / 10; - if (roundDigit > 5) { - isRoundPortionHalfEven = true; - } else if (roundDigit == 5) { - boolean exactlyOneHalf; - if (adjustedRoundingPoint - 1 == 0) { - // Just examine the middle and lower words. - exactlyOneHalf = (fast1 == 0 && fast0 == 0); - } else { - exactlyOneHalf = (fast2 % roundDivisor == 0 && fast1 == 0 && fast0 == 0); - } - - // When fraction is exactly 0.5 and lowest new digit is odd, go towards even. - if (exactlyOneHalf) { - isRoundPortionHalfEven = (fast2Scaled % 2 == 1); - } else { - isRoundPortionHalfEven = true; - } - } else { - isRoundPortionHalfEven = false; - } - } - } - return isRoundPortionHalfEven; - } - - private static void doClearRoundIntegerPortionAndAddOne( - long fast0, long fast1, long fast2, - int absRoundPower, - FastHiveDecimal fastResult) { - - long result0; - long result1; - long result2; - - if (absRoundPower < LONGWORD_DECIMAL_DIGITS) { - - // Lowest word gets integer rounding. - - // Clear rounding portion in lower longword and add 1 at right scale (roundMultiplyFactor). - - final long roundFactor = powerOfTenTable[absRoundPower]; - - final long r0 = - ((fast0 / roundFactor) * roundFactor) - + roundFactor; - result0 = r0 % MULTIPLER_LONGWORD_DECIMAL; - final long r1 = - fast1 - + r0 / MULTIPLER_LONGWORD_DECIMAL; - result1 = r1 % MULTIPLER_LONGWORD_DECIMAL; - result2 = - fast2 - + r1 / MULTIPLER_LONGWORD_DECIMAL; - - } else if (absRoundPower < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - // Middle word gets integer rounding; lower longword is cleared. - - final int adjustedAbsPower = absRoundPower - LONGWORD_DECIMAL_DIGITS; - - // Clear rounding portion in middle longword and add 1 at right scale (roundMultiplyFactor); - // lower longword result is 0; - - final long roundFactor = powerOfTenTable[adjustedAbsPower]; - - result0 = 0; - final long r1 = - ((fast1 / roundFactor) * roundFactor) - + roundFactor; - result1 = r1 % MULTIPLER_LONGWORD_DECIMAL; - result2 = - fast2 - + r1 / MULTIPLER_LONGWORD_DECIMAL; - - } else { - - // High word gets integer rounding; middle and lower longwords are cleared. - - final int adjustedAbsPower = absRoundPower - TWO_X_LONGWORD_DECIMAL_DIGITS; - - // Clear rounding portion in high longword and add 1 at right scale (roundMultiplyFactor); - // middle and lower longwords result is 0; - - final long roundFactor = powerOfTenTable[adjustedAbsPower]; - - result0 = 0; - result1 = 0; - result2 = - ((fast2 / roundFactor) * roundFactor) - + roundFactor; - - } - - fastResult.fast0 = result0; - fastResult.fast1 = result1; - fastResult.fast2 = result2; - } - - private static void doClearRoundIntegerPortion( - long fast0, long fast1, long fast2, - int absRoundPower, - FastHiveDecimal fastResult) { - - long result0; - long result1; - long result2; - - if (absRoundPower < LONGWORD_DECIMAL_DIGITS) { - - // Lowest word gets integer rounding. - - // Clear rounding portion in lower longword and add 1 at right scale (roundMultiplyFactor). - - final long roundFactor = powerOfTenTable[absRoundPower]; - // final long roundMultiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - absRoundPower]; - - result0 = - ((fast0 / roundFactor) * roundFactor); - result1 = fast1; - result2 = fast2; - - } else if (absRoundPower < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - // Middle word gets integer rounding; lower longword is cleared. - - final int adjustedAbsPower = absRoundPower - LONGWORD_DECIMAL_DIGITS; - - // Clear rounding portion in middle longword and add 1 at right scale (roundMultiplyFactor); - // lower longword result is 0; - - final long roundFactor = powerOfTenTable[adjustedAbsPower]; - - result0 = 0; - result1 = - ((fast1 / roundFactor) * roundFactor); - result2 = fast2; - - } else { - - // High word gets integer rounding; middle and lower longwords are cleared. - - final int adjustedAbsPower = absRoundPower - TWO_X_LONGWORD_DECIMAL_DIGITS; - - // Clear rounding portion in high longword and add 1 at right scale (roundMultiplyFactor); - // middle and lower longwords result is 0; - - final long roundFactor = powerOfTenTable[adjustedAbsPower]; - - result0 = 0; - result1 = 0; - result2 = - ((fast2 / roundFactor) * roundFactor); - - } - - fastResult.fast0 = result0; - fastResult.fast1 = result1; - fastResult.fast2 = result2; - } - - /** - * Fast decimal integer part rounding ROUND_UP. - * - * ceiling(12400.8302, -2) = 12500 // E.g. Positive case FAST_ROUND_CEILING - * rr rrrr - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param fastIntegerDigitCount the number of integer digits - * @param fastScale the scale of the number - * @param roundPower the power to round to - * @param fastResult an object to reuse - * @return was the operation successful - */ - public static boolean fastRoundIntegerUp( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int roundPower, - FastHiveDecimal fastResult) { - - /* - * Basic algorithm: - * - * 1. Determine if rounding part is non-zero for rounding. - * 2. Scale away fractional digits if present. - * 3. If rounding, clear integer rounding portion and add 1. - * - */ - - if (roundPower >= 0) { - throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")"); - } - - final int absRoundPower = -roundPower; - if (fastIntegerDigitCount < absRoundPower) { - - // Above decimal. - return false; - } - - final int roundingPoint = absRoundPower + fastScale; - if (roundingPoint > HiveDecimal.MAX_PRECISION) { - - // Value becomes null for rounding beyond. - return false; - } - - // First, determine whether rounding is necessary based on rounding point, which is inside - // integer part. And, get rid of any fractional digits. The result scale will be 0. - // - boolean isRoundPortionAllZeroes = - isRoundPortionAllZeroes( - fast0, fast1, fast2, - roundingPoint); - - // If necessary, divide and multiply to get rid of fractional digits. - if (fastScale == 0) { - fastResult.fast0 = fast0; - fastResult.fast1 = fast1; - fastResult.fast2 = fast2; - } else { - doFastScaleDown( - fast0, fast1, fast2, - /* scaleDown */ fastScale, - fastResult); - } - - // The fractional digits are gone; when rounding, clear remaining round digits and add 1. - if (!isRoundPortionAllZeroes) { - - doClearRoundIntegerPortionAndAddOne( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - absRoundPower, - fastResult); - } - - if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { - fastResult.fastSignum = 0; - fastResult.fastIntegerDigitCount = 0; - fastResult.fastScale = 0; - } else { - fastResult.fastSignum = fastSignum; - fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult); - fastResult.fastScale = 0; - } - - return true; - } - - /** - * Fast decimal scale down by factor of 10 with rounding ROUND_DOWN. - * - * The fraction being scaled away is thrown away. - * - * The signum will be updated if the result is 0, otherwise the original sign is unchanged. - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param fastIntegerDigitCount the number of integer digits - * @param fastScale the scale of the number - * @param roundPower the power to round to - * @param fastResult an object to reuse - * @return was the operation successful? - */ - public static boolean fastRoundIntegerDown( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int roundPower, - FastHiveDecimal fastResult) { - - /* - * Basic algorithm: - * - * 1. Scale away fractional digits if present. - * 2. Clear integer rounding portion. - * - */ - - if (roundPower >= 0) { - throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")"); - } - - final int absRoundPower = -roundPower; - if (fastIntegerDigitCount < absRoundPower) { - - // Zero result. - fastResult.fastReset(); - return true; - } - - final int roundingPoint = absRoundPower + fastScale; - if (roundingPoint > HiveDecimal.MAX_PRECISION) { - - // Value becomes zero for rounding beyond. - fastResult.fastReset(); - return true; - } - - // If necessary, divide and multiply to get rid of fractional digits. - if (fastScale == 0) { - fastResult.fast0 = fast0; - fastResult.fast1 = fast1; - fastResult.fast2 = fast2; - } else { - doFastScaleDown( - fast0, fast1, fast2, - /* scaleDown */ fastScale, - fastResult); - } - - // The fractional digits are gone; clear remaining round digits. - doClearRoundIntegerPortion( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - absRoundPower, - fastResult); - - if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { - fastResult.fastIntegerDigitCount = 0; - fastResult.fastScale = 0; - } else { - fastResult.fastSignum = 0; - fastResult.fastSignum = fastSignum; - fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult); - fastResult.fastScale = 0; - } - - return true; - } - - /** - * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP. - * - * When the fraction being scaled away is >= 0.5, the add 1. - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param fastIntegerDigitCount the number of integer digits - * @param fastScale the scale of the number - * @param roundPower the power to round to - * @param fastResult an object to reuse - * @return was the operation successful? - */ - public static boolean fastRoundIntegerHalfUp( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int roundPower, - FastHiveDecimal fastResult) { - - /* - * Basic algorithm: - * - * 1. Determine if rounding digit is >= 5 for rounding. - * 2. Scale away fractional digits if present. - * 3. If rounding, clear integer rounding portion and add 1. - * - */ - - if (roundPower >= 0) { - throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")"); - } - - final int absRoundPower = -roundPower; - if (fastIntegerDigitCount < absRoundPower) { - - // Zero result. - fastResult.fastReset(); - return true; - } - - final int roundingPoint = absRoundPower + fastScale; - if (roundingPoint > HiveDecimal.MAX_PRECISION) { - - // Value becomes zero for rounding beyond. - fastResult.fastReset(); - return true; - } - - // First, determine whether rounding is necessary based on rounding point, which is inside - // integer part. And, get rid of any fractional digits. The result scale will be 0. - // - boolean isRoundPortionHalfUp = - isRoundPortionHalfUp( - fast0, fast1, fast2, - roundingPoint); - - // If necessary, divide and multiply to get rid of fractional digits. - if (fastScale == 0) { - fastResult.fast0 = fast0; - fastResult.fast1 = fast1; - fastResult.fast2 = fast2; - } else { - doFastScaleDown( - fast0, fast1, fast2, - /* scaleDown */ fastScale, - fastResult); - } - - // The fractional digits are gone; when rounding, clear remaining round digits and add 1. - if (isRoundPortionHalfUp) { - - doClearRoundIntegerPortionAndAddOne( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - absRoundPower, - fastResult); - } else { - - doClearRoundIntegerPortion( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - absRoundPower, - fastResult); - } - - if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { - fastResult.fastSignum = 0; - fastResult.fastIntegerDigitCount = 0; - fastResult.fastScale = 0; - } else { - fastResult.fastSignum = fastSignum; - fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult); - fastResult.fastScale = 0; - } - - return true; - } - - /** - * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_EVEN. - * - * When the fraction being scaled away is exactly 0.5, then round and add 1 only if aaa. - * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1. - * Otherwise, throw away fraction. - * - * The signum will be updated if the result is 0, otherwise the original sign is unchanged. - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param fastIntegerDigitCount the number of integer digits - * @param fastScale the scale of the number - * @param roundPower the power to round to - * @param fastResult an object to reuse - * @return was the operation successful? - */ - public static boolean fastRoundIntegerHalfEven( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int roundPower, - FastHiveDecimal fastResult) { - - /* - * Basic algorithm: - * - * 1. Determine if rounding part meets banker's rounding rules for rounding. - * 2. Scale away fractional digits if present. - * 3. If rounding, clear integer rounding portion and add 1. - * - */ - - if (roundPower >= 0) { - throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")"); - } - - final int absRoundPower = -roundPower; - if (fastIntegerDigitCount < absRoundPower) { - - // Zero result. - fastResult.fastReset(); - } - - final int roundingPoint = absRoundPower + fastScale; - if (roundingPoint > HiveDecimal.MAX_PRECISION) { - - // Value becomes zero for rounding beyond. - fastResult.fastReset(); - return true; - } - - // First, determine whether rounding is necessary based on rounding point, which is inside - // integer part. And, get rid of any fractional digits. The result scale will be 0. - // - boolean isRoundPortionHalfEven = - isRoundPortionHalfEven( - fast0, fast1, fast2, - roundingPoint); - - // If necessary, divide and multiply to get rid of fractional digits. - if (fastScale == 0) { - fastResult.fast0 = fast0; - fastResult.fast1 = fast1; - fastResult.fast2 = fast2; - } else { - doFastScaleDown( - fast0, fast1, fast2, - /* scaleDown */ fastScale, - fastResult); - } - - // The fractional digits are gone; when rounding, clear remaining round digits and add 1. - if (isRoundPortionHalfEven) { - - doClearRoundIntegerPortionAndAddOne( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - absRoundPower, - fastResult); - } else { - - doClearRoundIntegerPortion( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - absRoundPower, - fastResult); - } - - if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { - fastResult.fastSignum = 0; - fastResult.fastIntegerDigitCount = 0; - fastResult.fastScale = 0; - } else { - fastResult.fastSignum = fastSignum; - fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult); - fastResult.fastScale = 0; - } - - return true; - } - - /** - * Fast decimal scale down by factor of 10 and do not allow rounding. - * - * When the fraction being scaled away is non-zero, return false. - * - * The signum will be updated if the result is 0, otherwise the original sign - * is unchanged. - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param scaleDown the digits to scale down by - * @param fastResult an object to reuse - * @return was the operation successful? - */ - public static boolean fastScaleDownNoRound( - int fastSignum, long fast0, long fast1, long fast2, - int scaleDown, - FastHiveDecimal fastResult) { - if (scaleDown < 1 || scaleDown >= THREE_X_LONGWORD_DECIMAL_DIGITS - 1) { - throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 3*16 - 1 (scaleDown " + scaleDown + ")"); - } - - // Adjust all longs using power 10 division/remainder. - long result0; - long result1; - long result2; - if (scaleDown < LONGWORD_DECIMAL_DIGITS) { - - // Part of lowest word survives. - - final long divideFactor = powerOfTenTable[scaleDown]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; - - final long throwAwayFraction = fast0 % divideFactor; - - if (throwAwayFraction != 0) { - return false; - } - result0 = - fast0 / divideFactor - + ((fast1 % divideFactor) * multiplyFactor); - result1 = - fast1 / divideFactor - + ((fast2 % divideFactor) * multiplyFactor); - result2 = - fast2 / divideFactor; - - } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - // Throw away lowest word. - - final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS; - - final long divideFactor = powerOfTenTable[adjustedScaleDown]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; - - boolean isThrowAwayFractionZero; - if (adjustedScaleDown == 0) { - isThrowAwayFractionZero = (fast0 == 0); - } else { - final long throwAwayFraction = fast1 % divideFactor; - isThrowAwayFractionZero = (throwAwayFraction == 0 && fast0 == 0); - } - - if (!isThrowAwayFractionZero) { - return false; - } - result0 = - fast1 / divideFactor - + ((fast2 % divideFactor) * multiplyFactor); - result1 = - fast2 / divideFactor; - result2 = 0; - - } else { - - // Throw away middle and lowest words. - - final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS; - - final long divideFactor = powerOfTenTable[adjustedScaleDown]; - - boolean isThrowAwayFractionZero; - if (adjustedScaleDown == 0) { - isThrowAwayFractionZero = (fast0 == 0 && fast1 == 0); - } else { - final long throwAwayFraction = fast2 % divideFactor; - isThrowAwayFractionZero = (throwAwayFraction == 0 && fast0 == 0 && fast1 == 0); - } - - if (!isThrowAwayFractionZero) { - return false; - } - result0 = - fast2 / divideFactor; - result1 = 0; - result2 = 0; - } - - if (result0 == 0 && result1 == 0 && result2 == 0) { - fastResult.fastReset(); - } else { - fastResult.fastSignum = fastSignum; - fastResult.fast0 = result0; - fastResult.fast1 = result1; - fastResult.fast2 = result2; - } - - return true; - } - - /** - * Fast decimal scale down by factor of 10 with rounding ROUND_UP. - * - * When the fraction being scaled away is non-zero, the add 1. - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param scaleDown the number of integer digits to scale - * @param fastResult an object to reuse - * @return was the operation successfule? - */ - public static boolean fastRoundFractionalUp( - int fastSignum, long fast0, long fast1, long fast2, - int scaleDown, - FastHiveDecimal fastResult) { - if (scaleDown < 1 || scaleDown > HiveDecimal.MAX_SCALE) { - throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < " + HiveDecimal.MAX_SCALE + " (scaleDown " + scaleDown + ")"); - } - - if (scaleDown == HiveDecimal.MAX_SCALE) { - - // Examine all digits being thrown away to determine if result is 0 or 1. - if (fast0 == 0 && fast1 == 0 && fast2 == 0) { - - // Zero result. - fastResult.fastReset(); - } else { - fastResult.fastSet(fastSignum, /* fast0 */ 1, 0, 0, /* fastIntegerDigitCount */ 1, 0); - } - return true; - } - - boolean isRoundPortionAllZeroes = - isRoundPortionAllZeroes( - fast0, fast1, fast2, - scaleDown); - - doFastScaleDown( - fast0, fast1, fast2, - scaleDown, - fastResult); - - if (!isRoundPortionAllZeroes) { - final long r0 = fastResult.fast0 + 1; - fastResult.fast0 = - r0 % MULTIPLER_LONGWORD_DECIMAL; - final long r1 = - fastResult.fast1 - + r0 / MULTIPLER_LONGWORD_DECIMAL; - fastResult.fast1 = - r1 % MULTIPLER_LONGWORD_DECIMAL; - fastResult.fast2 = - fastResult.fast2 - + r1 / MULTIPLER_LONGWORD_DECIMAL; - } - - if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { - fastResult.fastSignum = 0; - fastResult.fastIntegerDigitCount = 0; - fastResult.fastScale = 0; - } else { - fastResult.fastSignum = fastSignum; - } - - return (fastResult.fast2 <= MAX_HIGHWORD_DECIMAL); - } - - /** - * Fast decimal scale down by factor of 10 with rounding ROUND_DOWN. - * - * The fraction being scaled away is thrown away. - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param scaleDown the number of integer digits to scale - * @param fastResult an object to reuse - */ - public static void fastRoundFractionalDown( - int fastSignum, long fast0, long fast1, long fast2, - int scaleDown, - FastHiveDecimal fastResult) { - if (scaleDown < 1 || scaleDown > HiveDecimal.MAX_SCALE) { - throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 38 (scaleDown " + scaleDown + ")"); - } - - if (scaleDown == HiveDecimal.MAX_SCALE) { - - // Complete fractional digits shear off. Zero result. - fastResult.fastReset(); - return; - } - - doFastScaleDown( - fast0, fast1, fast2, - scaleDown, - fastResult); - - if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { - fastResult.fastSignum = 0; - fastResult.fastIntegerDigitCount = 0; - fastResult.fastScale = 0; - } else { - fastResult.fastSignum = fastSignum; - } - } - - /** - * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP. - * - * When the fraction being scaled away is >= 0.5, the add 1. - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param scaleDown the number of integer digits to scale - * @param fastResult an object to reuse - * @return was the operation successfule? - */ - public static boolean fastRoundFractionalHalfUp( - int fastSignum, long fast0, long fast1, long fast2, - int scaleDown, - FastHiveDecimal fastResult) { - if (fastSignum == 0) { - throw new IllegalArgumentException("Unexpected zero value"); - } - if (scaleDown < 1 || scaleDown > HiveDecimal.MAX_SCALE) { - throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 38 (scaleDown " + scaleDown + ")"); - } - - if (scaleDown == HiveDecimal.MAX_SCALE) { - - // Check highest digit for rounding. - final long roundDigit = fast2 / powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1]; - if (roundDigit < 5) { - - // Zero result. - fastResult.fastReset(); - } else { - fastResult.fastSet(fastSignum, /* fast0 */ 1, 0, 0, /* fastIntegerDigitCount */ 1, 0); - } - return true; - } - - boolean isRoundPortionHalfUp = - isRoundPortionHalfUp( - fast0, fast1, fast2, - scaleDown); - - doFastScaleDown( - fast0, fast1, fast2, - scaleDown, - fastResult); - - if (isRoundPortionHalfUp) { - final long r0 = fastResult.fast0 + 1; - fastResult.fast0 = - r0 % MULTIPLER_LONGWORD_DECIMAL; - final long r1 = - fastResult.fast1 - + r0 / MULTIPLER_LONGWORD_DECIMAL; - fastResult.fast1 = - r1 % MULTIPLER_LONGWORD_DECIMAL; - fastResult.fast2 = - fastResult.fast2 - + r1 / MULTIPLER_LONGWORD_DECIMAL; - } - - if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { - fastResult.fastSignum = 0; - fastResult.fastIntegerDigitCount = 0; - fastResult.fastScale = 0; - } else { - fastResult.fastSignum = fastSignum; - } - - return (fastResult.fast2 <= MAX_HIGHWORD_DECIMAL); - } - - /** - * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP. - * - * When the fraction being scaled away is >= 0.5, the add 1. - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param fast3 word 3 - * @param fast4 word 4 - * @param scaleDown the number of integer digits to scale - * @param fastResult an object to reuse - * @return was the operation successfule? - */ - public static boolean fastRoundFractionalHalfUp5Words( - int fastSignum, long fast0, long fast1, long fast2, long fast3, long fast4, - int scaleDown, - FastHiveDecimal fastResult) { - - // Adjust all longs using power 10 division/remainder. - long result0; - long result1; - long result2; - long result3; - long result4; - if (scaleDown < LONGWORD_DECIMAL_DIGITS) { - - // Part of lowest word survives. - - // Divide down just before scaleDown to get round digit. - final long withRoundDigit = fast0 / powerOfTenTable[scaleDown - 1]; - final long roundDigit = withRoundDigit % 10; - - final long divideFactor = powerOfTenTable[scaleDown]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; - - if (roundDigit < 5) { - result0 = - withRoundDigit / 10 - + ((fast1 % divideFactor) * multiplyFactor); - result1 = - fast1 / divideFactor - + ((fast2 % divideFactor) * multiplyFactor); - result2 = - + fast2 / divideFactor - + ((fast3 % divideFactor) * multiplyFactor); - result3 = - fast3 / divideFactor - + ((fast4 % divideFactor) * multiplyFactor); - result4 = - fast4 / divideFactor; - } else { - // Add rounding and handle carry. - final long r0 = - withRoundDigit / 10 - + ((fast1 % divideFactor) * multiplyFactor) - + 1; - result0 = r0 % MULTIPLER_LONGWORD_DECIMAL; - final long r1 = - fast1 / divideFactor - + ((fast2 % divideFactor) * multiplyFactor) - + r0 / MULTIPLER_LONGWORD_DECIMAL; - result1 = r1 % MULTIPLER_LONGWORD_DECIMAL; - final long r2 = - fast2 / divideFactor + - + ((fast3 % divideFactor) * multiplyFactor) - + r1 / MULTIPLER_LONGWORD_DECIMAL; - result2 = r2 % MULTIPLER_LONGWORD_DECIMAL; - final long r3 = - fast3 / divideFactor - + ((fast4 % divideFactor) * multiplyFactor) - + r2 / MULTIPLER_LONGWORD_DECIMAL; - result3 = r3 % MULTIPLER_LONGWORD_DECIMAL; - result4 = - fast4 / divideFactor + - r3 % MULTIPLER_LONGWORD_DECIMAL; - } - } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - // Throw away lowest word. - - final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS; - - long roundDigit; - long fast1Scaled; - if (adjustedScaleDown == 0) { - // Grab round digit from lowest word. - roundDigit = fast0 / (MULTIPLER_LONGWORD_DECIMAL / 10); - fast1Scaled = fast1; - } else { - // Divide down just before scaleDown to get round digit. - final long withRoundDigit = fast1 / powerOfTenTable[adjustedScaleDown - 1]; - roundDigit = withRoundDigit % 10; - fast1Scaled = withRoundDigit / 10; - } - - final long divideFactor = powerOfTenTable[adjustedScaleDown]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; - - if (roundDigit < 5) { - result0 = - fast1Scaled - + ((fast2 % divideFactor) * multiplyFactor); - result1 = - fast2 / divideFactor - + ((fast3 % divideFactor) * multiplyFactor); - result2 = - fast3 / divideFactor - + ((fast4 % divideFactor) * multiplyFactor); - result3 = - fast4 / divideFactor; - } else { - // Add rounding and handle carry. - final long r0 = - fast1Scaled - + ((fast2 % divideFactor) * multiplyFactor) - + 1; - result0 = r0 % MULTIPLER_LONGWORD_DECIMAL; - final long r1 = - fast2 / divideFactor - + ((fast3 % divideFactor) * multiplyFactor) - + r0 / MULTIPLER_LONGWORD_DECIMAL; - result1 = r1 % MULTIPLER_LONGWORD_DECIMAL; - final long r2 = - fast3 / divideFactor - + ((fast4 % divideFactor) * multiplyFactor) - + r1 / MULTIPLER_LONGWORD_DECIMAL; - result2 = r2 % MULTIPLER_LONGWORD_DECIMAL; - result3 = - fast4 / divideFactor - + r2 / MULTIPLER_LONGWORD_DECIMAL; - } - result4 = 0; - } else { - - // Throw away middle and lowest words. - - final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS; - - long roundDigit; - long fast2Scaled; - if (adjustedScaleDown == 0) { - // Grab round digit from middle word. - roundDigit = fast1 / (MULTIPLER_LONGWORD_DECIMAL / 10); - fast2Scaled = fast2; - } else { - // Divide down just before scaleDown to get round digit. - final long withRoundDigit = fast2 / powerOfTenTable[adjustedScaleDown - 1]; - roundDigit = withRoundDigit % 10; - fast2Scaled = withRoundDigit / 10; - } - - final long divideFactor = powerOfTenTable[adjustedScaleDown]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; - - if (roundDigit < 5) { - result0 = - fast2Scaled - + ((fast3 % divideFactor) * multiplyFactor); - result1 = - fast3 / divideFactor - + ((fast4 % divideFactor) * multiplyFactor); - result2 = - fast4 / divideFactor; - } else { - // Add rounding. - final long r0 = - fast2Scaled - + ((fast3 % divideFactor) * multiplyFactor) - + 1; - result0 = r0 % MULTIPLER_LONGWORD_DECIMAL; - final long r1 = - fast3 / divideFactor - + ((fast4 % divideFactor) * multiplyFactor) - + r0 / MULTIPLER_LONGWORD_DECIMAL; - result1 = r1 % MULTIPLER_LONGWORD_DECIMAL; - result2 = - fast4 / divideFactor - + r1 / MULTIPLER_LONGWORD_DECIMAL; - } - result3 = 0; - result4 = 0; - } - - if (result4 != 0 || result3 != 0) { - throw new RuntimeException("Unexpected overflow into result3 or result4"); - } - if (result0 == 0 && result1 == 0 && result2 == 0) { - fastResult.fastReset(); - } - fastResult.fastSignum = fastSignum; - fastResult.fast0 = result0; - fastResult.fast1 = result1; - fastResult.fast2 = result2; - - return (result2 <= MAX_HIGHWORD_DECIMAL); - } - - /** - * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_EVEN. - * - * When the fraction being scaled away is exactly 0.5, then round and add 1 only if aaa. - * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1. - * Otherwise, throw away fraction. - * - * @param fastSignum the sign (-1, 0, or +1) - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param scaleDown the number of integer digits to scale - * @param fastResult an object to reuse - * @return was the operation successfule? - */ - public static boolean fastRoundFractionalHalfEven( - int fastSignum, long fast0, long fast1, long fast2, - int scaleDown, - FastHiveDecimal fastResult) { - if (scaleDown < 1 || scaleDown > HiveDecimal.MAX_SCALE) { - throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 38 (scaleDown " + scaleDown + ")"); - } - - if (scaleDown == HiveDecimal.MAX_SCALE) { - - // Check for rounding. - final long roundDivisor = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1]; - final long withRoundDigit = fast2 / roundDivisor; - final long roundDigit = withRoundDigit % 10; - final long fast2Scaled = withRoundDigit / 10; - boolean shouldRound; - if (roundDigit > 5) { - shouldRound = true; - } else if (roundDigit == 5) { - boolean exactlyOneHalf = (fast2Scaled == 0 && fast1 == 0 && fast0 == 0); - if (exactlyOneHalf) { - // Round to even 0. - shouldRound = false; - } else { - shouldRound = true; - } - } else { - shouldRound = false; - } - if (!shouldRound) { - - // Zero result. - fastResult.fastReset(); - } else { - fastResult.fastSet(fastSignum, /* fast0 */ 1, 0, 0, /* fastIntegerDigitCount */ 1, 0); - } - return true; - } - - boolean isRoundPortionHalfEven = - isRoundPortionHalfEven( - fast0, fast1, fast2, - scaleDown); - - doFastScaleDown( - fast0, fast1, fast2, - scaleDown, - fastResult); - - if (isRoundPortionHalfEven) { - final long r0 = fastResult.fast0 + 1; - fastResult.fast0 = - r0 % MULTIPLER_LONGWORD_DECIMAL; - final long r1 = - fastResult.fast1 - + r0 / MULTIPLER_LONGWORD_DECIMAL; - fastResult.fast1 = - r1 % MULTIPLER_LONGWORD_DECIMAL; - fastResult.fast2 = - fastResult.fast2 - + r1 / MULTIPLER_LONGWORD_DECIMAL; - } - - if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { - fastResult.fastSignum = 0; - fastResult.fastIntegerDigitCount = 0; - fastResult.fastScale = 0; - } else { - fastResult.fastSignum = fastSignum; - } - - return (fastResult.fast2 <= MAX_HIGHWORD_DECIMAL); - } - - public static void doFastScaleDown( - FastHiveDecimal fastDec, - int scaleDown, - FastHiveDecimal fastResult) { - doFastScaleDown( - fastDec.fast0, fastDec.fast1, fastDec.fast2, - scaleDown, - fastResult); - } - - //************************************************************************************************ - // Decimal Scale Up/Down. - - /** - * Fast decimal scale down by factor of 10 with NO rounding. - * - * The signum will be updated if the result is 0, otherwise the original sign is unchanged. - * - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param scaleDown the number of integer digits to scale - * @param fastResult an object to reuse - */ - public static void doFastScaleDown( - long fast0, long fast1, long fast2, - int scaleDown, - FastHiveDecimal fastResult) { - - // Adjust all longs using power 10 division/remainder. - long result0; - long result1; - long result2; - if (scaleDown < LONGWORD_DECIMAL_DIGITS) { - - // Part of lowest word survives. - - final long divideFactor = powerOfTenTable[scaleDown]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; - - result0 = - fast0 / divideFactor - + ((fast1 % divideFactor) * multiplyFactor); - result1 = - fast1 / divideFactor - + ((fast2 % divideFactor) * multiplyFactor); - result2 = - fast2 / divideFactor; - - } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - // Throw away lowest word. - - final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS; - - final long divideFactor = powerOfTenTable[adjustedScaleDown]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; - - result0 = - fast1 / divideFactor - + ((fast2 % divideFactor) * multiplyFactor); - result1 = - fast2 / divideFactor; - result2 = 0; - - } else { - - // Throw away middle and lowest words. - - final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS; - - result0 = - fast2 / powerOfTenTable[adjustedScaleDown]; - result1 = 0; - result2 = 0; - - } - - if (result0 == 0 && result1 == 0 && result2 == 0) { - fastResult.fastSignum = 0; - } - fastResult.fast0 = result0; - fastResult.fast1 = result1; - fastResult.fast2 = result2; - } - - public static boolean fastScaleUp( - FastHiveDecimal fastDec, - int scaleUp, - FastHiveDecimal fastResult) { - return - fastScaleUp( - fastDec.fast0, fastDec.fast1, fastDec.fast2, - scaleUp, - fastResult); - } - - /** - * Fast decimal scale up by factor of 10. - * @param fast0 word 0 of the internal representation - * @param fast1 word 1 - * @param fast2 word 2 - * @param scaleUp the number of integer digits to scale up by - * @param fastResult an object to reuse - * @return was the operation successfule? - */ - public static boolean fastScaleUp( - long fast0, long fast1, long fast2, - int scaleUp, - FastHiveDecimal fastResult) { - if (scaleUp < 1 || scaleUp >= HiveDecimal.MAX_SCALE) { - throw new IllegalArgumentException("Expecting scaleUp > 0 and scaleUp < 38"); - } - - long result0; - long result1; - long result2; - - // Each range checks for overflow first, then moves digits. - if (scaleUp < HIGHWORD_DECIMAL_DIGITS) { - // Need to check if there are overflow digits in the high word. - - final long overflowFactor = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - scaleUp]; - if (fast2 / overflowFactor != 0) { - return false; - } - - final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleUp]; - final long multiplyFactor = powerOfTenTable[scaleUp]; - - result2 = - fast2 * multiplyFactor - + fast1 / divideFactor; - result1 = - (fast1 % divideFactor) * multiplyFactor - + fast0 / divideFactor; - result0 = - (fast0 % divideFactor) * multiplyFactor; - } else if (scaleUp < HIGHWORD_DECIMAL_DIGITS + LONGWORD_DECIMAL_DIGITS) { - // High word must be zero. Check for overflow digits in middle word. - - if (fast2 != 0) { - return false; - } - - final int adjustedScaleUp = scaleUp - HIGHWORD_DECIMAL_DIGITS; - - final int middleDigits = LONGWORD_DECIMAL_DIGITS - adjustedScaleUp; - final long overflowFactor = powerOfTenTable[middleDigits]; - if (fast1 / overflowFactor != 0) { - return false; - } - - if (middleDigits < HIGHWORD_DECIMAL_DIGITS) { - // Must fill high word from both middle and lower longs. - - final int highWordMoreDigits = HIGHWORD_DECIMAL_DIGITS - middleDigits; - final long multiplyFactor = powerOfTenTable[highWordMoreDigits]; - - final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - highWordMoreDigits]; - - result2 = - fast1 * multiplyFactor - + fast0 / divideFactor; - result1 = - (fast0 % divideFactor) * multiplyFactor; - result0 = 0; - } else if (middleDigits == HIGHWORD_DECIMAL_DIGITS) { - // Fill high long from middle long, and middle long from lower long. - - result2 = fast1; - result1 = fast0; - result0 = 0; - } else { - // Fill high long from some of middle long. - - final int keepMiddleDigits = middleDigits - HIGHWORD_DECIMAL_DIGITS; - final long divideFactor = powerOfTenTable[keepMiddleDigits]; - final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - keepMiddleDigits]; - - result2 = - fast1 / divideFactor; - result1 = - (fast1 % divideFactor) * multiplyFactor - + fast0 / divideFactor; - result0 = - (fast0 % divideFactor) * multiplyFactor; - } - } else { - // High and middle word must be zero. Check for overflow digits in lower word. - - if (fast2 != 0 || fast1 != 0) { - return false; - } - - final int adjustedScaleUp = scaleUp - HIGHWORD_DECIMAL_DIGITS - LONGWORD_DECIMAL_DIGITS; - - final int lowerDigits = LONGWORD_DECIMAL_DIGITS - adjustedScaleUp; - final long overflowFactor = powerOfTenTable[lowerDigits]; - if (fast0 / overflowFactor != 0) { - return false; - } - - if (lowerDigits < HIGHWORD_DECIMAL_DIGITS) { - // Must fill high word from both middle and lower longs. - - final int highWordMoreDigits = HIGHWORD_DECIMAL_DIGITS - lowerDigits; - final long multiplyFactor = powerOfTenTable[highWordMoreDigits]; - - final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - highWordMoreDigits]; - - result2 = - fast0 * multiplyFactor; - result1 = 0; - result0 = 0; - } else if (lowerDigits == HIGHWORD_DECIMAL_DIGITS) { - // Fill high long from lower long. - - result2 = fast0; - result1 = 0; - result0 = 0; - } else { - // Fill high long and middle from some of lower long. - - final int keepLowerDigits = lowerDigits - HIGHWORD_DECIMAL_DIGITS; - final long keepLowerDivideFactor = powerOfTenTable[keepLowerDigits]; - final long keepLowerMultiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - keepLowerDigits]; - - result2 = - fast0 / keepLowerDivideFactor; - result1 = - (fast0 % keepLowerDivideFactor) * keepLowerMultiplyFactor; - result0 = 0; - } - } - - if (result0 == 0 && result1 == 0 && result2 == 0) { - fastResult.fastSignum = 0; - } - fastResult.fast0 = result0; - fastResult.fast1 = result1; - fastResult.fast2 = result2; - - return true; - } - - //************************************************************************************************ - // Decimal Precision / Trailing Zeroes. - - public static int fastLongWordTrailingZeroCount( - long longWord) { - - if (longWord == 0) { - return LONGWORD_DECIMAL_DIGITS; - } - - long factor = 10; - for (int i = 0; i < LONGWORD_DECIMAL_DIGITS; i++) { - if (longWord % factor != 0) { - return i; - } - factor *= 10; - } - return 0; - } - - public static int fastHighWordTrailingZeroCount( - long longWord) { - - if (longWord == 0) { - return HIGHWORD_DECIMAL_DIGITS; - } - - long factor = 10; - for (int i = 0; i < HIGHWORD_DECIMAL_DIGITS; i++) { - if (longWord % factor != 0) { - return i; - } - factor *= 10; - } - return 0; - } - - public static int fastLongWordPrecision( - long longWord) { - - if (longWord == 0) { - return 0; - } - // Search like binary search to minimize comparisons. - if (longWord > 99999999L) { - if (longWord > 999999999999L) { - if (longWord > 99999999999999L) { - if (longWord > 999999999999999L) { - return 16; - } else { - return 15; - } - } else { - if (longWord > 9999999999999L) { - return 14; - } else { - return 13; - } - } - } else { - if (longWord > 9999999999L) { - if (longWord > 99999999999L) { - return 12; - } else { - return 11; - } - } else { - if (longWord > 999999999L) { - return 10; - } else { - return 9; - } - } - } - } else { - if (longWord > 9999L) { - if (longWord > 999999L) { - if (longWord > 9999999L) { - return 8; - } else { - return 7; - } - } else { - if (longWord > 99999L) { - return 6; - } else { - return 5; - } - } - } else { - if (longWord > 99L) { - if (longWord > 999L) { - return 4; - } else { - return 3; - } - } else { - if (longWord > 9L) { - return 2; - } else { - return 1; - } - } - } - } - } - - public static int fastHighWordPrecision( - long longWord) { - - if (longWord == 0) { - return 0; - } - // 6 highword digits. - if (longWord > 999L) { - if (longWord > 9999L) { - if (longWord > 99999L) { - return 6; - } else { - return 5; - } - } else { - return 4; - } - } else { - if (longWord > 99L) { - return 3; - } else { - if (longWord > 9L) { - return 2; - } else { - return 1; - } - } - } - } - - public static int fastSqlPrecision( - FastHiveDecimal fastDec) { - return - fastSqlPrecision( - fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2, - fastDec.fastIntegerDigitCount, fastDec.fastScale); - } - - public static int fastSqlPrecision( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - - if (fastSignum == 0) { - return 1; - } - - final int rawPrecision = - fastRawPrecision(fastSignum, fast0, fast1, fast2); - - if (rawPrecision < fastScale) { - // This can happen for numbers less than 0.1 - // For 0.001234: rawPrecision=4, scale=6 - // In this case, we'll set the type to have the same precision as the scale. - return fastScale; - } - return rawPrecision; - } - - public static int fastRawPrecision( - FastHiveDecimal fastDec) { - return - fastRawPrecision( - fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2); - } - - public static int fastRawPrecision( - int fastSignum, long fast0, long fast1, long fast2) { - - if (fastSignum == 0) { - return 0; - } - - int precision; - - if (fast2 != 0) { - - // 6 highword digits. - precision = TWO_X_LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(fast2); - - } else if (fast1 != 0) { - - // Check fast1. - precision = LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fast1); - - - } else { - - // Check fast0. - precision = fastLongWordPrecision(fast0); - - } - - return precision; - } - - // Determine if all digits below a power is zero. - // The lowest digit is power = 0. - public static boolean isAllZeroesBelow( - int fastSignum, long fast0, long fast1, long fast2, - int power) { - if (power < 0 || power > HiveDecimal.MAX_SCALE) { - throw new IllegalArgumentException("Expecting power >= 0 and power <= 38"); - } - - if (fastSignum == 0) { - return true; - } - - if (power >= TWO_X_LONGWORD_DECIMAL_DIGITS) { - if (fast0 != 0 || fast1 != 0) { - return false; - } - final int adjustedPower = power - TWO_X_LONGWORD_DECIMAL_DIGITS; - if (adjustedPower == 0) { - return true; - } - long remainder = fast2 % powerOfTenTable[adjustedPower]; - return (remainder == 0); - } else if (power >= LONGWORD_DECIMAL_DIGITS) { - if (fast0 != 0) { - return false; - } - final int adjustedPower = power - LONGWORD_DECIMAL_DIGITS; - if (adjustedPower == 0) { - return true; - } - long remainder = fast1 % powerOfTenTable[adjustedPower]; - return (remainder == 0); - } else { - if (power == 0) { - return true; - } - long remainder = fast0 % powerOfTenTable[power]; - return (remainder == 0); - } - } - - public static boolean fastExceedsPrecision( - long fast0, long fast1, long fast2, - int precision) { - - if (precision <= 0) { - return true; - } else if (precision >= HiveDecimal.MAX_PRECISION) { - return false; - } - final int precisionLessOne = precision - 1; - - // 0 (lowest), 1 (middle), or 2 (high). - int wordNum = precisionLessOne / LONGWORD_DECIMAL_DIGITS; - - int digitInWord = precisionLessOne % LONGWORD_DECIMAL_DIGITS; - - final long overLimitInWord = powerOfTenTable[digitInWord + 1] - 1; - - if (wordNum == 0) { - if (digitInWord < LONGWORD_DECIMAL_DIGITS - 1) { - if (fast0 > overLimitInWord) { - return true; - } - } - return (fast1 != 0 || fast2 != 0); - } else if (wordNum == 1) { - if (digitInWord < LONGWORD_DECIMAL_DIGITS - 1) { - if (fast1 > overLimitInWord) { - return true; - } - } - return (fast2 != 0); - } else { - // We've eliminated the highest digit already with HiveDecimal.MAX_PRECISION check above. - return (fast2 > overLimitInWord); - } - } - - public static int fastTrailingDecimalZeroCount( - long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - if (fastScale < 0 || fastScale > HiveDecimal.MAX_SCALE) { - throw new IllegalArgumentException("Expecting scale >= 0 and scale <= 38"); - } - if (fastScale == 0) { - return 0; - } - - final int lowerLongwordDigits = Math.min(fastScale, LONGWORD_DECIMAL_DIGITS); - if (lowerLongwordDigits < LONGWORD_DECIMAL_DIGITS || fast0 != 0) { - long factor = 10; - for (int i = 0; i < lowerLongwordDigits; i++) { - if (fast0 % factor != 0) { - return i; - } - factor *= 10; - } - if (lowerLongwordDigits < LONGWORD_DECIMAL_DIGITS) { - return fastScale; - } - } - if (fastScale == LONGWORD_DECIMAL_DIGITS) { - return fastScale; - } - final int middleLongwordDigits = Math.min(fastScale - LONGWORD_DECIMAL_DIGITS, LONGWORD_DECIMAL_DIGITS); - if (middleLongwordDigits < LONGWORD_DECIMAL_DIGITS || fast1 != 0) { - long factor = 10; - for (int i = 0; i < middleLongwordDigits; i++) { - if (fast1 % factor != 0) { - return LONGWORD_DECIMAL_DIGITS + i; - } - factor *= 10; - } - if (middleLongwordDigits < LONGWORD_DECIMAL_DIGITS) { - return fastScale; - } - } - if (fastScale == TWO_X_LONGWORD_DECIMAL_DIGITS) { - return fastScale; - } - final int highLongwordDigits = fastScale - TWO_X_LONGWORD_DECIMAL_DIGITS; - if (highLongwordDigits < HIGHWORD_DECIMAL_DIGITS || fast2 != 0) { - long factor = 10; - for (int i = 0; i < highLongwordDigits; i++) { - if (fast2 % factor != 0) { - return TWO_X_LONGWORD_DECIMAL_DIGITS + i; - } - factor *= 10; - } - } - return fastScale; - } - - public static FastCheckPrecisionScaleStatus fastCheckPrecisionScale( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int maxPrecision, int maxScale) { - - if (fastSignum == 0) { - return FastCheckPrecisionScaleStatus.NO_CHANGE; - } - final int maxIntegerDigitCount = maxPrecision - maxScale; - if (fastIntegerDigitCount > maxIntegerDigitCount) { - return FastCheckPrecisionScaleStatus.OVERFLOW; - } - if (fastScale > maxScale) { - return FastCheckPrecisionScaleStatus.UPDATE_SCALE_DOWN; - } - return FastCheckPrecisionScaleStatus.NO_CHANGE; - } - - public static boolean fastUpdatePrecisionScale( - final int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int maxPrecision, int maxScale, FastCheckPrecisionScaleStatus status, - FastHiveDecimal fastResult) { - - switch (status) { - case UPDATE_SCALE_DOWN: - { - fastResult.fastSignum = fastSignum; - - // Throw away lower digits. - if (!fastRoundFractionalHalfUp( - fastSignum, fast0, fast1, fast2, - fastScale - maxScale, - fastResult)) { - return false; - } - - fastResult.fastScale = maxScale; - - // CONSIDER: For now, recompute integerDigitCount... - fastResult.fastIntegerDigitCount = - Math.max(0, fastRawPrecision(fastResult) - fastResult.fastScale); - - // And, round up may cause us to exceed our precision/scale... - final int maxIntegerDigitCount = maxPrecision - maxScale; - if (fastResult.fastIntegerDigitCount > maxIntegerDigitCount) { - return false; - } - - // Scaling down may have opened up trailing zeroes... - final int trailingZeroCount = - fastTrailingDecimalZeroCount( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - fastResult.fastIntegerDigitCount, fastResult.fastScale); - if (trailingZeroCount > 0) { - // Scale down again. - doFastScaleDown( - fastResult, - trailingZeroCount, - fastResult); - fastResult.fastScale -= trailingZeroCount; - } - } - break; - default: - throw new RuntimeException("Unexpected fast check precision scale status " + status); - } - - return true; - } - - //************************************************************************************************ - // Decimal Addition / Subtraction. - - public static boolean doAddSameScaleSameSign( - FastHiveDecimal fastLeft, - FastHiveDecimal fastRight, - FastHiveDecimal fastResult) { - return - doAddSameScaleSameSign( - /* resultSignum */ fastLeft.fastSignum, - fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, - fastRight.fast0, fastRight.fast1, fastRight.fast2, - fastResult); - } - - public static boolean doAddSameScaleSameSign( - int resultSignum, - long left0, long left1, long left2, - long right0, long right1, long right2, - FastHiveDecimal fastResult) { - - long result0; - long result1; - long result2; - - final long r0 = left0 + right0; - result0 = - r0 % MULTIPLER_LONGWORD_DECIMAL; - final long r1 = - left1 - + right1 - + r0 / MULTIPLER_LONGWORD_DECIMAL; - result1 = - r1 % MULTIPLER_LONGWORD_DECIMAL; - result2 = - left2 - + right2 - + r1 / MULTIPLER_LONGWORD_DECIMAL; - - if (result0 == 0 && result1 == 0 && result2 == 0) { - fastResult.fastReset(); - } else { - fastResult.fastSignum = resultSignum; - fastResult.fast0 = result0; - fastResult.fast1 = result1; - fastResult.fast2 = result2; - } - - return (result2 <= MAX_HIGHWORD_DECIMAL); - } - - public static boolean doSubtractSameScaleNoUnderflow( - int resultSignum, - FastHiveDecimal fastLeft, - FastHiveDecimal fastRight, - FastHiveDecimal fastResult) { - return - doSubtractSameScaleNoUnderflow( - resultSignum, - fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, - fastRight.fast0, fastRight.fast1, fastRight.fast2, - fastResult); - } - - public static boolean doSubtractSameScaleNoUnderflow( - int resultSignum, - long left0, long left1, long left2, - long right0, long right1, long right2, - FastHiveDecimal fastResult) { - - long result0; - long result1; - long result2; - - final long r0 = left0 - right0; - long r1; - if (r0 < 0) { - result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; - r1 = left1 - right1 - 1; - } else { - result0 = r0; - r1 = left1 - right1; - } - if (r1 < 0) { - result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; - result2 = left2 - right2 - 1; - } else { - result1 = r1; - result2 = left2 - right2; - } - if (result2 < 0) { - return false; - } - - if (result0 == 0 && result1 == 0 && result2 == 0) { - fastResult.fastReset(); - } else { - fastResult.fastSignum = resultSignum; - fastResult.fast0 = result0; - fastResult.fast1 = result1; - fastResult.fast2 = result2; - } - - return true; - } - - public static boolean doSubtractSameScaleNoUnderflow( - long left0, long left1, long left2, - long right0, long right1, long right2, - long[] result) { - - long result0; - long result1; - long result2; - - final long r0 = left0 - right0; - long r1; - if (r0 < 0) { - result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; - r1 = left1 - right1 - 1; - } else { - result0 = r0; - r1 = left1 - right1; - } - if (r1 < 0) { - result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; - result2 = left2 - right2 - 1; - } else { - result1 = r1; - result2 = left2 - right2; - } - if (result2 < 0) { - return false; - } - - result[0] = result0; - result[1] = result1; - result[2] = result2; - - return true; - } - - private static boolean doAddSameScale( - int leftSignum, long leftFast0, long leftFast1, long leftFast2, - int rightSignum, long rightFast0, long rightFast1, long rightFast2, - int scale, - FastHiveDecimal fastResult) { - - if (leftSignum == rightSignum) { - if (!doAddSameScaleSameSign( - /* resultSignum */ leftSignum, - leftFast0, leftFast1, leftFast2, - rightFast0, rightFast1, rightFast2, - fastResult)) { - // Handle overflow precision issue. - if (scale > 0) { - if (!fastRoundFractionalHalfUp( - fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, - 1, - fastResult)) { - return false; - } - scale--; - } else { - // Overflow. - return false; - } - } - fastResult.fastScale = scale; - } else { - // Just compare the magnitudes (i.e. signums set to 1). - int compareTo = - fastCompareTo( - 1, - leftFast0, leftFast1, leftFast2, 0, - 1, - rightFast0, rightFast1, rightFast2, 0); - if (compareTo == 0) { - // They cancel each other. - fastResult.fastReset(); - return true; - } - - if (compareTo == 1) { - if (!doSubtractSameScaleNoUnderflow( - /* resultSignum */ leftSignum, - leftFast0, leftFast1, leftFast2, - rightFast0, rightFast1, rightFast2, - fastResult)) { - throw new RuntimeException("Unexpected underflow"); - } - } else { - if (!doSubtractSameScaleNoUnderflow( - /* resultSignum */ rightSignum, - rightFast0, rightFast1, rightFast2, - leftFast0, leftFast1, leftFast2, - fastResult)) { - throw new RuntimeException("Unexpected underflow"); - } - } - fastResult.fastScale = scale; - } - - if (fastResult.fastSignum != 0) { - final int precision = fastRawPrecision(fastResult); - fastResult.fastIntegerDigitCount = Math.max(0, precision - fastResult.fastScale); - } - - final int resultTrailingZeroCount = - fastTrailingDecimalZeroCount( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - fastResult.fastIntegerDigitCount, fastResult.fastScale); - if (resultTrailingZeroCount > 0) { - doFastScaleDown( - fastResult, - resultTrailingZeroCount, - fastResult); - if (fastResult.fastSignum == 0) { - fastResult.fastScale = 0; - } else { - fastResult.fastScale -= resultTrailingZeroCount; - } - } - - return true; - } - - /** - * Handle the common logic at the end of fastAddDifferentScale / fastSubtractDifferentScale that - * takes the 5 intermediate result words and fits them into the 3 longword fast result. - * - */ - private static boolean doFinishAddSubtractDifferentScale( - long result0, long result1, long result2, long result3, long result4, - int resultScale, - FastHiveDecimal fastResult) { - - int precision; - if (result4 != 0) { - precision = FOUR_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result4); - } else if (result3 != 0) { - precision = THREE_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result3); - } else if (result2 != 0) { - precision = TWO_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result2); - } else if (result1 != 0) { - precision = LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result1); - } else { - precision = fastLongWordPrecision(result0); - } - - if (precision > HiveDecimal.MAX_PRECISION){ - final int scaleDown = precision - HiveDecimal.MAX_PRECISION; - - resultScale -= scaleDown; - if (resultScale < 0) { - // No room. - return false; - } - if (!fastRoundFractionalHalfUp5Words( - 1, result0, result1, result2, result3, result4, - scaleDown, - fastResult)) { - // Handle overflow precision issue. - if (resultScale > 0) { - if (!fastRoundFractionalHalfUp( - fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, - 1, - fastResult)) { - throw new RuntimeException("Unexpected overflow"); - } - if (fastResult.fastSignum == 0) { - return true; - } - resultScale--; - } else { - return false; - } - } - - precision = fastRawPrecision(1, fastResult.fast0, fastResult.fast1, fastResult.fast2); - - // Stick back into result variables... - result0 = fastResult.fast0; - result1 = fastResult.fast1; - result2 = fastResult.fast2; - } - - // Caller will set signum. - fastResult.fastSignum = 1; - fastResult.fast0 = result0; - fastResult.fast1 = result1; - fastResult.fast2 = result2; - fastResult.fastIntegerDigitCount = Math.max(0, precision - resultScale); - fastResult.fastScale = resultScale; - - final int resultTrailingZeroCount = - fastTrailingDecimalZeroCount( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - fastResult.fastIntegerDigitCount, fastResult.fastScale); - if (resultTrailingZeroCount > 0) { - doFastScaleDown( - fastResult, - resultTrailingZeroCount, - fastResult); - if (fastResult.fastSignum == 0) { - fastResult.fastScale = 0; - } else { - fastResult.fastScale -= resultTrailingZeroCount; - } - } - - return true; - } - - /** - * Handle decimal subtraction when the values have different scales. - */ - private static boolean fastSubtractDifferentScale( - long leftFast0, long leftFast1, long leftFast2, - int leftIntegerDigitCount, int leftScale, - long rightFast0, long rightFast1, long rightFast2, - int rightIntegerDigitCount, int rightScale, - FastHiveDecimal fastResult) { - - int diffScale; - int resultScale; - - long result0 = 0; - long result1 = 0; - long result2 = 0; - long result3 = 0; - long result4 = 0; - - // Since subtraction is not commutative, we can must subtract in the order passed in. - if (leftScale > rightScale) { - - // Since left has a longer digit tail and it doesn't move; we will shift the right digits - // as we do our addition into the result. - - diffScale = leftScale - rightScale; - resultScale = leftScale; - - if (diffScale < LONGWORD_DECIMAL_DIGITS) { - - final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - diffScale]; - final long multiplyFactor = powerOfTenTable[diffScale]; - - final long r0 = - leftFast0 - - (rightFast0 % divideFactor) * multiplyFactor; - long r1; - if (r0 < 0) { - result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; - r1 = - leftFast1 - - rightFast0 / divideFactor - - (rightFast1 % divideFactor) * multiplyFactor - - 1; - } else { - result0 = r0; - r1 = - leftFast1 - - rightFast0 / divideFactor - - (rightFast1 % divideFactor) * multiplyFactor; - } - long r2; - if (r1 < 0) { - result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; - r2 = - leftFast2 - - rightFast1 / divideFactor - - (rightFast2 % divideFactor) * multiplyFactor - - 1; - } else { - result1 = r1; - r2 = - leftFast2 - - rightFast1 / divideFactor - - (rightFast2 % divideFactor) * multiplyFactor; - } - long r3; - if (r2 < 0) { - result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; - r3 = - -(rightFast2 / divideFactor) - - 1; - } else { - result2 = r2; - r3 = - -(rightFast2 / divideFactor); - } - long r4; - if (r3 < 0) { - result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; - r4 = - 1; - } else { - result3 = r3; - r4 = 0; - } - if (r4 != 0) { - throw new RuntimeException("Unexpected underflow"); - } - - } else if (diffScale == LONGWORD_DECIMAL_DIGITS){ - - result0 = leftFast0; - final long r1 = - leftFast1 - - rightFast0; - long r2; - if (r1 < 0) { - result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; - r2 = - leftFast2 - - rightFast1 - - 1; - } else { - result1 = r1; - r2 = - leftFast2 - - rightFast1; - } - long r3; - if (r2 < 0) { - result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; - r3 = - -rightFast2 - - 1; - } else { - result2 = r2; - r3 = - -rightFast2; - } - if (r3 != 0) { - throw new RuntimeException("Unexpected underflow"); - } - - - } else if (diffScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - final long divideFactor = powerOfTenTable[TWO_X_LONGWORD_DECIMAL_DIGITS - diffScale]; - final long multiplyFactor = powerOfTenTable[diffScale - LONGWORD_DECIMAL_DIGITS]; - - result0 = leftFast0; - final long r1 = - leftFast1 - -(rightFast0 % divideFactor) * multiplyFactor; - long r2; - if (r1 < 0) { - result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; - r2 = - leftFast2 - - rightFast0 / divideFactor - - (rightFast1 % divideFactor) * multiplyFactor - - 1; - } else { - result1 = r1; - r2 = - leftFast2 - - rightFast0 / divideFactor - - (rightFast1 % divideFactor) * multiplyFactor; - } - long r3; - if (r2 < 0) { - result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; - r3 = - - rightFast1 / divideFactor - - (rightFast2 % divideFactor) * multiplyFactor - - 1; - } else { - result2 = r2; - r3 = - - rightFast1 / divideFactor - - (rightFast2 % divideFactor) * multiplyFactor; - } - long r4; - if (r3 < 0) { - result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; - r4 = - - rightFast2 / divideFactor - - 1; - } else { - result3 = r3; - r4 = - - rightFast2 / divideFactor; - } - long r5; - if (r4 < 0) { - result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; - r5 = - 1; - } else { - result4 = r4; - r5 = 0; - } - if (r5 != 0) { - throw new RuntimeException("Unexpected underflow"); - } - - } else if (diffScale == TWO_X_LONGWORD_DECIMAL_DIGITS) { - - result0 = leftFast0; - result1 = leftFast1; - final long r2 = - leftFast2 - - rightFast0; - long r3; - if (r2 < 0) { - result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; - r3 = - - rightFast1 - - 1; - } else { - result2 = r2; - r3 = - - rightFast1; - } - long r4; - if (r3 < 0) { - result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; - r4 = - -rightFast2 - - 1; - } else { - result3 = r3; - r4 = - -rightFast2; - } - long r5; - if (r4 < 0) { - result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; - r5 = - 1; - } else { - result4 = r4; - r5 = 0; - } - if (r5 != 0) { - throw new RuntimeException("Unexpected underflow"); - } - - } else { - - final long divideFactor = powerOfTenTable[THREE_X_LONGWORD_DECIMAL_DIGITS - diffScale]; - final long multiplyFactor = powerOfTenTable[diffScale - TWO_X_LONGWORD_DECIMAL_DIGITS]; - - result0 = leftFast0; - result1 = leftFast1; - final long r2 = - leftFast2 - - (rightFast0 % divideFactor) * multiplyFactor; - long r3; - if (r2 < 0) { - result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; - r3 = - - (rightFast0 / divideFactor) - - (rightFast1 % divideFactor) * multiplyFactor - - 1; - } else { - result2 = r2; - r3 = - - (rightFast0 / divideFactor) - - (rightFast1 % divideFactor) * multiplyFactor; - } - long r4; - if (r3 < 0) { - result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; - r4 = - - (rightFast1 / divideFactor) - - (rightFast2 % divideFactor) * multiplyFactor - - 1; - } else { - result3 = r3; - r4 = - - (rightFast1 / divideFactor) - - (rightFast2 % divideFactor) * multiplyFactor; - } - long r5; - if (r4 < 0) { - result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; - r5 = - - (rightFast2 / divideFactor) - - 1; - } else { - result4 = r4; - r5 = - - (rightFast2 / divideFactor); - } - if (r5 != 0) { - throw new RuntimeException("Unexpected underflow"); - } - } - } else { - - // Since right has a longer digit tail and it doesn't move; we will shift the left digits - // as we do our addition into the result. - - diffScale = rightScale - leftScale; - resultScale = rightScale; - - if (diffScale < LONGWORD_DECIMAL_DIGITS) { - - final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - diffScale]; - final long multiplyFactor = powerOfTenTable[diffScale]; - - final long r0 = - (leftFast0 % divideFactor) * multiplyFactor - - rightFast0; - long r1; - if (r0 < 0) { - result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; - r1 = - leftFast0 / divideFactor - + (leftFast1 % divideFactor) * multiplyFactor - - rightFast1 - - 1; - } else { - result0 = r0; - r1 = - leftFast0 / divideFactor - + (leftFast1 % divideFactor) * multiplyFactor - - rightFast1; - } - long r2; - if (r1 < 0) { - result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; - r2 = - leftFast1 / divideFactor - + (leftFast2 % divideFactor) * multiplyFactor - - rightFast2 - - 1; - } else { - result1 = r1; - r2 = - leftFast1 / divideFactor - + (leftFast2 % divideFactor) * multiplyFactor - - rightFast2; - } - long r3; - if (r2 < 0) { - result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; - r3 = - leftFast2 / divideFactor - - 1; - } else { - result2 = r2; - r3 = - leftFast2 / divideFactor; - } - long r4; - if (r3 < 0) { - result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; - r4 = - 1; - } else { - result3 = r3; - r4 = 0; - } - if (r4 != 0) { - throw new RuntimeException("Unexpected underflow"); - } - - } else if (diffScale == LONGWORD_DECIMAL_DIGITS){ - - final long r0 = - - rightFast0; - long r1; - if (r0 < 0) { - result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; - r1 = - leftFast0 - - rightFast1 - - 1; - } else { - result0 = r0; - r1 = - leftFast0 - - rightFast1; - } - long r2; - if (r1 < 0) { - result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; - r2 = - leftFast1 - - rightFast2 - - 1; - } else { - result1 = r1; - r2 = - leftFast1 - - rightFast2; - } - long r3; - if (r2 < 0) { - result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; - r3 = - leftFast2 - - 1; - } else { - result2 = r2; - r3 = - leftFast2; - } - long r4; - if (r3 < 0) { - result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; - r4 = - 1; - } else { - result3 = r3; - r4 = 0; - } - if (r4 != 0) { - throw new RuntimeException("Unexpected underflow"); - } - - } else if (diffScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - final long divideFactor = powerOfTenTable[TWO_X_LONGWORD_DECIMAL_DIGITS - diffScale]; - final long multiplyFactor = powerOfTenTable[diffScale - LONGWORD_DECIMAL_DIGITS]; - - final long r0 = - - rightFast0; - long r1; - if (r0 < 0) { - result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; - r1 = - (leftFast0 % divideFactor) * multiplyFactor - - rightFast1 - - 1; - } else { - result0 = r0; - r1 = - (leftFast0 % divideFactor) * multiplyFactor - - rightFast1; - } - long r2; - if (r1 < 0) { - result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; - r2 = - leftFast0 / divideFactor - + (leftFast1 % divideFactor) * multiplyFactor - - rightFast2 - - 1; - } else { - result1 = r1; - r2 = - leftFast0 / divideFactor - + (leftFast1 % divideFactor) * multiplyFactor - - rightFast2; - } - long r3; - if (r2 < 0) { - result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; - r3 = - leftFast1 / divideFactor - + (leftFast2 % divideFactor) * multiplyFactor - - 1; - } else { - result2 = r2; - r3 = - leftFast1 / divideFactor - + (leftFast2 % divideFactor) * multiplyFactor; - } - long r4; - if (r3 < 0) { - result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; - r4 = - leftFast2 / divideFactor - - 1; - } else { - result3 = r3; - r4 = - leftFast2 / divideFactor; - } - if (r4 < 0) { - result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; - } else { - result4 = r4; - } - - } else if (diffScale == TWO_X_LONGWORD_DECIMAL_DIGITS) { - - final long r0 = - - rightFast0; - long r1; - if (r0 < 0) { - result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; - r1 = - - rightFast1 - - 1; - } else { - result0 = r0; - r1 = - - rightFast1; - } - long r2; - if (r1 < 0) { - result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; - r2 = - leftFast0 - - rightFast2 - - 1; - } else { - result1 = r1; - r2 = - leftFast0 - - rightFast2; - } - long r3; - if (r2 < 0) { - result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; - r3 = - leftFast1 - - 1; - } else { - result2 = r2; - r3 = - leftFast1; - } - long r4; - if (r3 < 0) { - result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; - r4 = - leftFast2 - - 1; - } else { - result3 = r3; - r4 = - leftFast2; - } - long r5; - if (r4 < 0) { - result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; - r5 = - 1; - } else { - result4 = r4; - r5 = 0; - } - if (r5 != 0) { - throw new RuntimeException("Unexpected underflow"); - } - - } else { - - final long divideFactor = powerOfTenTable[THREE_X_LONGWORD_DECIMAL_DIGITS - diffScale]; - final long multiplyFactor = powerOfTenTable[diffScale - TWO_X_LONGWORD_DECIMAL_DIGITS]; - - final long r0 = - - rightFast0; - long r1; - if (r0 < 0) { - result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; - r1 = - - rightFast1 - - 1; - } else { - result0 = r0; - r1 = - - rightFast1; - } - long r2; - if (r1 < 0) { - result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; - r2 = - (leftFast0 % divideFactor) * multiplyFactor - - rightFast2 - - 1; - } else { - result1 = r1; - r2 = - (leftFast0 % divideFactor) * multiplyFactor - - rightFast2; - } - long r3; - if (r2 < 0) { - result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; - r3 = - leftFast0 / divideFactor - + (leftFast1 % divideFactor) * multiplyFactor - - 1; - } else { - result2 = r2; - r3 = - leftFast0 / divideFactor - + (leftFast1 % divideFactor) * multiplyFactor; - } - long r4; - if (r3 < 0) { - result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; - r4 = - leftFast1 / divideFactor - + (leftFast2 % divideFactor) * multiplyFactor - - 1; - } else { - result3 = r3; - r4 = - leftFast1 / divideFactor - + (leftFast2 % divideFactor) * multiplyFactor; - } - long r5; - if (r4 < 0) { - result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; - r5 = - leftFast2 / divideFactor - - 1; - } else { - result4 = r4; - r5 = - leftFast2 / divideFactor; - } - if (r5 != 0) { - throw new RuntimeException("Unexpected underflow"); - } - } - } - - return - doFinishAddSubtractDifferentScale( - result0, result1, result2, result3, result4, - resultScale, - fastResult); - } - - /** - * Handle decimal addition when the values have different scales. - */ - private static boolean fastAddDifferentScale( - long leftFast0, long leftFast1, long leftFast2, - int leftIntegerDigitCount, int leftScale, - long rightFast0, long rightFast1, long rightFast2, - int rightIntegerDigitCount, int rightScale, - FastHiveDecimal fastResult) { - - // Arrange so result* has a longer digit tail and it lines up; we will shift the shift* digits - // as we do our addition and them into the result. - long result0; - long result1; - long result2; - - long shift0; - long shift1; - long shift2; - - int diffScale; - int resultScale; - - // Since addition is commutative, we can add in any order. - if (leftScale > rightScale) { - - result0 = leftFast0; - result1 = leftFast1; - result2 = leftFast2; - - shift0 = rightFast0; - shift1 = rightFast1; - shift2 = rightFast2; - - diffScale = leftScale - rightScale; - resultScale = leftScale; - } else { - - result0 = rightFast0; - result1 = rightFast1; - result2 = rightFast2; - - shift0 = leftFast0; - shift1 = leftFast1; - shift2 = leftFast2; - - diffScale = rightScale - leftScale; - resultScale = rightScale; - } - - long result3 = 0; - long result4 = 0; - - if (diffScale < LONGWORD_DECIMAL_DIGITS) { - - final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - diffScale]; - final long multiplyFactor = powerOfTenTable[diffScale]; - - final long r0 = - result0 - + (shift0 % divideFactor) * multiplyFactor; - result0 = - r0 % MULTIPLER_LONGWORD_DECIMAL; - final long r1 = - result1 - + shift0 / divideFactor - + (shift1 % divideFactor) * multiplyFactor - + r0 / MULTIPLER_LONGWORD_DECIMAL; - result1 = - r1 % MULTIPLER_LONGWORD_DECIMAL; - final long r2 = - result2 - + shift1 / divideFactor - + (shift2 % divideFactor) * multiplyFactor - + r1 / MULTIPLER_LONGWORD_DECIMAL; - result2 = - r2 % MULTIPLER_LONGWORD_DECIMAL; - final long r3 = - shift2 / divideFactor - + r2 / MULTIPLER_LONGWORD_DECIMAL; - result3 = - r3 % MULTIPLER_LONGWORD_DECIMAL; - - } else if (diffScale == LONGWORD_DECIMAL_DIGITS){ - - final long r1 = - result1 - + shift0; - result1 = - r1 % MULTIPLER_LONGWORD_DECIMAL; - final long r2 = - result2 - + shift1 - + r1 / MULTIPLER_LONGWORD_DECIMAL; - result2 = - r2 % MULTIPLER_LONGWORD_DECIMAL; - final long r3 = - shift2 - + r2 / MULTIPLER_LONGWORD_DECIMAL; - result3 = - r3 % MULTIPLER_LONGWORD_DECIMAL; - result4 = r3 / MULTIPLER_LONGWORD_DECIMAL; - - } else if (diffScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { - - final long divideFactor = powerOfTenTable[TWO_X_LONGWORD_DECIMAL_DIGITS - diffScale]; - final long multiplyFactor = powerOfTenTable[diffScale - LONGWORD_DECIMAL_DIGITS]; - - final long r1 = - result1 - + (shift0 % divideFactor) * multiplyFactor; - result1 = - r1 % MULTIPLER_LONGWORD_DECIMAL; - final long r2 = - result2 - + shift0 / divideFactor - + (shift1 % divideFactor) * multiplyFactor - + r1 / MULTIPLER_LONGWORD_DECIMAL; - result2 = - r2 % MULTIPLER_LONGWORD_DECIMAL; - final long r3 = - shift1 / divideFactor - + (shift2 % divideFactor) * multiplyFactor - + r2 / MULTIPLER_LONGWORD_DECIMAL; - result3 = - r3 % MULTIPLER_LONGWORD_DECIMAL; - final long r4 = - shift2 / divideFactor - + r3 / MULTIPLER_LONGWORD_DECIMAL; - result4 = - r4 % MULTIPLER_LONGWORD_DECIMAL; - - } else if (diffScale == TWO_X_LONGWORD_DECIMAL_DIGITS) { - - final long r2 = - result2 - + shift0; - result2 = - r2 % MULTIPLER_LONGWORD_DECIMAL; - final long r3 = - shift1 - + r2 / MULTIPLER_LONGWORD_DECIMAL; - result3 = - r3 % MULTIPLER_LONGWORD_DECIMAL; - final long r4 = - shift2 - + r3 / MULTIPLER_LONGWORD_DECIMAL; - result4 = - r4 % MULTIPLER_LONGWORD_DECIMAL; - - } else { - - final long divideFactor = powerOfTenTable[THREE_X_LONGWORD_DECIMAL_DIGITS - diffScale]; - final long multiplyFactor = powerOfTenTable[diffScale - TWO_X_LONGWORD_DECIMAL_DIGITS]; - - final long r2 = - result2 - + (shift0 % divideFactor) * multiplyFactor; - result2 = - r2 % MULTIPLER_LONGWORD_DECIMAL; - final long r3 = - shift0 / divideFactor - + (shift1 % divideFactor) * multiplyFactor - + r2 / MULTIPLER_LONGWORD_DECIMAL; - result3 = - r3 % MULTIPLER_LONGWORD_DECIMAL; - final long r4 = - shift1 / divideFactor - + (shift2 % divideFactor) * multiplyFactor - + r3 / MULTIPLER_LONGWORD_DECIMAL; - result4 = - r4 % MULTIPLER_LONGWORD_DECIMAL; - if (shift2 / divideFactor != 0) { - throw new RuntimeException("Unexpected overflow"); - } - - } - - return - doFinishAddSubtractDifferentScale( - result0, result1, result2, result3, result4, - resultScale, - fastResult); - } - - private static boolean doAddDifferentScale( - int leftSignum, long leftFast0, long leftFast1, long leftFast2, - int leftIntegerDigitCount, int leftScale, - int rightSignum, long rightFast0, long rightFast1, long rightFast2, - int rightIntegerDigitCount, int rightScale, - FastHiveDecimal fastResult) { - - if (leftSignum == rightSignum) { - if (!fastAddDifferentScale( - leftFast0, leftFast1, leftFast2, - leftIntegerDigitCount, leftScale, - rightFast0, rightFast1, rightFast2, - rightIntegerDigitCount, rightScale, - fastResult)) { - return false; - } - // Sign stays the same. - fastResult.fastSignum = leftSignum; - } else { - - // Just compare the magnitudes (i.e. signums set to 1). - int compareTo = - fastCompareTo( - 1, - leftFast0, leftFast1, leftFast2, leftScale, - 1, - rightFast0, rightFast1, rightFast2, rightScale); - if (compareTo == 0) { - // They cancel each other. - fastResult.fastSignum = 0; - fastResult.fast0 = 0; - fastResult.fast1 = 0; - fastResult.fast2 = 0; - fastResult.fastScale = 0; - return true; - } - - if (compareTo == 1) { - if (!fastSubtractDifferentScale( - leftFast0, leftFast1, leftFast2, - leftIntegerDigitCount, leftScale, - rightFast0, rightFast1, rightFast2, - rightIntegerDigitCount, rightScale, - fastResult)) { - throw new RuntimeException("Unexpected overflow"); - } - fastResult.fastSignum = leftSignum; - } else { - if (!fastSubtractDifferentScale( - rightFast0, rightFast1, rightFast2, - rightIntegerDigitCount, rightScale, - leftFast0, leftFast1, leftFast2, - leftIntegerDigitCount, leftScale, - fastResult)) { - throw new RuntimeException("Unexpected overflow"); - } - fastResult.fastSignum = rightSignum; - } - } - - final int resultTrailingZeroCount = - fastTrailingDecimalZeroCount( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - fastResult.fastIntegerDigitCount, fastResult.fastScale); - if (resultTrailingZeroCount > 0) { - doFastScaleDown( - fastResult, - resultTrailingZeroCount, - fastResult); - if (fastResult.fastSignum == 0) { - fastResult.fastScale = 0; - } else { - fastResult.fastScale -= resultTrailingZeroCount; - } - } - - return true; - } - - public static boolean fastAdd( - FastHiveDecimal fastLeft, - FastHiveDecimal fastRight, - FastHiveDecimal fastResult) { - return fastAdd( - fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, - fastLeft.fastIntegerDigitCount, fastLeft.fastScale, - fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, - fastRight.fastIntegerDigitCount, fastRight.fastScale, - fastResult); - } - - /** - * Add the two decimals. - * - * NOTE: Scale Determination for Addition/Subtraction - * - * One could take the Math.min of the scales and adjust the operand with the lower scale have a - * scale = higher scale. - * - * But this does not seem to work with decimals with widely varying scales as these: - * - * 598575157855521918987423259.94094 dec1 (int digits 27,scale 5) - * + 0.0000000000006711991169422033 dec2 (int digits 0, scale 28) - * - * Trying to make dec1 to have a scale of 28 (i.e. by adding trailing zeroes) would exceed - * MAX_PRECISION (int digits 27 + 28 > 38). - * - * In this example we need to make sure we have enough integer digit room in the result to - * handle dec1's digits. In order to maintain that, we will need to get rid of lower - * fractional digits of dec2. But when do we do that? - * - * OldHiveDecimal.add does the full arithmetic add with all the digits using BigDecimal and - * then adjusts the result to fit in MAX_PRECISION, etc. - * - * If we try to do pre-rounding dec2 it is problematic. We'd need to know if there is a carry in - * the arithmetic in order to know at which scale to do the rounding. This gets complicated. - * - * So, the simplest thing is to emulate what OldHiveDecimal does and do the full digit addition - * and then fit the result afterwards. - * - * @param leftSignum The left sign (-1, 0, or +1) - * @param leftFast0 The left word 0 of reprentation - * @param leftFast1 word 1 - * @param leftFast2 word 2 - * @param leftIntegerDigitCount The left number of integer digits - * @param leftScale the left scale - * @param rightSignum The right sign (-1, 0, or +1) - * @param rightFast0 The right word 0 of reprentation - * @param rightFast1 word 1 - * @param rightFast2 word 2 - * @param rightIntegerDigitCount The right number of integer digits - * @param rightScale the right scale - * @param fastResult an object to reuse - * @return True if the addition was successful; Otherwise, false is returned on overflow. - */ - public static boolean fastAdd( - int leftSignum, long leftFast0, long leftFast1, long leftFast2, - int leftIntegerDigitCount, int leftScale, - int rightSignum, long rightFast0, long rightFast1, long rightFast2, - int rightIntegerDigitCount, int rightScale, - FastHiveDecimal fastResult) { - - if (rightSignum == 0) { - fastResult.fastSet(leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale); - return true; - } - if (leftSignum == 0) { - fastResult.fastSet(rightSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale); - return true; - } - - if (leftScale == rightScale) { - return doAddSameScale( - leftSignum, leftFast0, leftFast1, leftFast2, - rightSignum, rightFast0, rightFast1, rightFast2, - leftScale, - fastResult); - } else { - return doAddDifferentScale( - leftSignum, leftFast0, leftFast1, leftFast2, - leftIntegerDigitCount, leftScale, - rightSignum, rightFast0, rightFast1, rightFast2, - rightIntegerDigitCount, rightScale, - fastResult); - } - } - - public static boolean fastSubtract( - FastHiveDecimal fastLeft, - FastHiveDecimal fastRight, - FastHiveDecimal fastResult) { - return fastSubtract( - fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, - fastLeft.fastIntegerDigitCount, fastLeft.fastScale, - fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, - fastRight.fastIntegerDigitCount, fastRight.fastScale, - fastResult); - } - - public static boolean fastSubtract( - int leftSignum, long leftFast0, long leftFast1, long leftFast2, - int leftIntegerDigitCount, int leftScale, - int rightSignum, long rightFast0, long rightFast1, long rightFast2, - int rightIntegerDigitCount, int rightScale, - FastHiveDecimal fastResult) { - - if (rightSignum == 0) { - fastResult.fastSet(leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale); - return true; - } - final int flippedDecSignum = (rightSignum == 1 ? -1 : 1); - if (leftSignum == 0) { - fastResult.fastSet(flippedDecSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale); - return true; - } - - if (leftScale == rightScale) { - return doAddSameScale( - leftSignum, leftFast0, leftFast1, leftFast2, - flippedDecSignum, rightFast0, rightFast1, rightFast2, - leftScale, - fastResult); - } else { - return doAddDifferentScale( - leftSignum, leftFast0, leftFast1, leftFast2, - leftIntegerDigitCount, leftScale, - flippedDecSignum, rightFast0, rightFast1, rightFast2, - rightIntegerDigitCount, rightScale, - fastResult); - } - } - - //************************************************************************************************ - // Decimal Multiply. - - private static boolean doMultiply( - int leftSignum, long leftFast0, long leftFast1, long leftFast2, - int leftIntegerDigitCount, int leftScale, - int rightSignum, long rightFast0, long rightFast1, long rightFast2, - int rightIntegerDigitCount, int rightScale, - FastHiveDecimal fastResult) { - - // Set signum before; if result is zero, fastMultiply will set signum to 0. - fastResult.fastSignum = (leftSignum == rightSignum ? 1 : -1); - int resultScale = leftScale + rightScale; - - /* - * For multiplicands with scale 0, trim trailing zeroes. - */ - if (leftScale == 0) { - - // Pretend like it has fractional digits so we can get the trailing zero count. - final int leftTrailingZeroCount = - fastTrailingDecimalZeroCount( - leftFast0, leftFast1, leftFast2, - 0, leftIntegerDigitCount); - if (leftTrailingZeroCount > 0) { - doFastScaleDown( - leftFast0, leftFast1, leftFast2, leftTrailingZeroCount, fastResult); - resultScale -= leftTrailingZeroCount; - leftFast0 = fastResult.fast0; - leftFast1 = fastResult.fast1; - leftFast2 = fastResult.fast2; - } - } - if (rightScale == 0) { - - // Pretend like it has fractional digits so we can get the trailing zero count. - final int rightTrailingZeroCount = - fastTrailingDecimalZeroCount( - rightFast0, rightFast1, rightFast2, - 0, rightIntegerDigitCount); - if (rightTrailingZeroCount > 0) { - doFastScaleDown( - rightFast0, rightFast1, rightFast2, rightTrailingZeroCount, fastResult); - resultScale -= rightTrailingZeroCount; - rightFast0 = fastResult.fast0; - rightFast1 = fastResult.fast1; - rightFast2 = fastResult.fast2; - } - } - - boolean largeOverflow = - !fastMultiply5x5HalfWords( - leftFast0, leftFast1, leftFast2, - rightFast0, rightFast1, rightFast2, - fastResult); - if (largeOverflow) { - return false; - } - - if (fastResult.fastSignum == 0) { - fastResult.fastScale = 0; - return true; - } - - if (resultScale < 0) { - if (-resultScale >= HiveDecimal.MAX_SCALE) { - return false; - } - if (!fastScaleUp( - fastResult.fast0, fastResult.fast1, fastResult.fast2, -resultScale, - fastResult)) { - return false; - } - resultScale = 0; - } - - int precision; - if (fastResult.fast2 != 0) { - precision = TWO_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast2); - } else if (fastResult.fast1 != 0) { - precision = LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast1); - } else { - precision = fastLongWordPrecision(fastResult.fast0); - } - - int integerDigitCount = Math.max(0, precision - resultScale); - if (integerDigitCount > HiveDecimal.MAX_PRECISION) { - // Integer is too large -- cannot recover by trimming fractional digits. - return false; - } - - if (precision > HiveDecimal.MAX_PRECISION || resultScale > HiveDecimal.MAX_SCALE) { - - // Trim off lower fractional digits but with NO ROUNDING. - - final int maxScale = HiveDecimal.MAX_SCALE - integerDigitCount; - final int scaleDown = resultScale - maxScale; - if (!fastScaleDownNoRound( - fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, - scaleDown, - fastResult)) { - // Round fractional must be 0. Not allowed to throw away digits. - return false; - } - resultScale -= scaleDown; - } - fastResult.fastScale = resultScale; - - // This assume no round up... - fastResult.fastIntegerDigitCount = integerDigitCount; - - if (fastResult.fastScale > HiveDecimal.MAX_SCALE) { - // We are not allowed to lose digits in multiply to be compatible with OldHiveDecimal - // behavior, so overflow. - // CONSIDER: Does it make sense to be so restrictive. If we just did repeated addition, - // it would succeed... - return false; - } - final int resultTrailingZeroCount = - fastTrailingDecimalZeroCount( - fastResult.fast0, fastResult.fast1, fastResult.fast2, - fastResult.fastIntegerDigitCount, fastResult.fastScale); - if (resultTrailingZeroCount > 0) { - doFastScaleDown( - fastResult, - resultTrailingZeroCount, - fastResult); - if (fastResult.fastSignum == 0) { - fastResult.fastScale = 0; - } else { - fastResult.fastScale -= resultTrailingZeroCount; - } - } - - return true; - } - - public static boolean fastMultiply5x5HalfWords( - FastHiveDecimal fastLeft, - FastHiveDecimal fastRight, - FastHiveDecimal fastResult) { - return - fastMultiply5x5HalfWords( - fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, - fastRight.fast0, fastRight.fast1, fastRight.fast2, - fastResult); - } - - /** - * Fast decimal multiplication on two decimals that have been already scaled and whose results - * will fit in 38 digits. - * - * The caller is responsible checking for overflow within the highword and determining - * if scale down appropriate. - * - * @return Returns false if the multiplication resulted in large overflow. Values in result are - * undefined in that case. - */ - public static boolean fastMultiply5x5HalfWords( - long left0, long left1, long left2, - long right0, long right1, long right2, - FastHiveDecimal fastResult) { - - long product; - - final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL; - final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL; - final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL; - final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL; - final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL; - - final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL; - final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL; - final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL; - final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL; - final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL; - - // v[0] - product = - halfRight0 * halfLeft0; - final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0]. - product = - halfRight0 - * halfLeft1 - + halfRight1 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[2] - product = - halfRight0 - * halfLeft2 - + halfRight1 - * halfLeft1 - + halfRight2 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[3] - product = - halfRight0 - * halfLeft3 - + halfRight1 - * halfLeft2 - + halfRight2 - * halfLeft1 - + halfRight3 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[4] - product = - halfRight0 - * halfLeft4 - + halfRight1 - * halfLeft3 - + halfRight2 - * halfLeft2 - + halfRight3 - * halfLeft1 - + halfRight4 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - - // v[5] is not calculated since high integer is always 0 for our decimals. - - // These remaining combinations below definitely result in overflow. - if ((halfRight4 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0 || halfLeft1 != 0)) - || (halfRight3 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0)) - || (halfRight2 != 0 && (halfLeft4 != 0 || halfLeft3 != 0)) - || (halfRight1 != 0 && halfLeft4 != 0)) { - return false; - } - - - final long result0 = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0; - final long result1 = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2; - final long result2 = product; - - if (result0 == 0 && result1 == 0 && result2 == 0) { - fastResult.fastSignum = 0; - } - fastResult.fast0 = result0; - fastResult.fast1 = result1; - fastResult.fast2 = result2; - - return true; - } - - public static boolean fastMultiplyFullInternal( - FastHiveDecimal fastLeft, - FastHiveDecimal fastRight, - long[] result) { - return - fastMultiplyFullInternal( - fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, - fastRight.fast0, fastRight.fast1, fastRight.fast2, - result); - } - - /** - * Fast decimal multiplication on two decimals that have been already scaled and whose results - * will fit in 38 digits. - * - * The caller is responsible checking for overflow within the highword and determining - * if scale down appropriate. - * - * @return Returns false if the multiplication resulted in large overflow. Values in result are - * undefined in that case. - */ - public static boolean fastMultiply5x5HalfWords( - long left0, long left1, long left2, - long right0, long right1, long right2, - long[] result) { - - long product; - - final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL; - final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL; - final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL; - final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL; - final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL; - - final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL; - final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL; - final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL; - final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL; - final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL; - - // v[0] - product = - halfRight0 * halfLeft0; - final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0]. - product = - halfRight0 - * halfLeft1 - + halfRight1 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[2] - product = - halfRight0 - * halfLeft2 - + halfRight1 - * halfLeft1 - + halfRight2 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[3] - product = - halfRight0 - * halfLeft3 - + halfRight1 - * halfLeft2 - + halfRight2 - * halfLeft1 - + halfRight3 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[4] - product = - halfRight0 - * halfLeft4 - + halfRight1 - * halfLeft3 - + halfRight2 - * halfLeft2 - + halfRight3 - * halfLeft1 - + halfRight4 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - - // v[5] is not calculated since high integer is always 0 for our decimals. - - // These remaining combinations below definitely result in overflow. - if ((halfRight4 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0 || halfLeft1 != 0)) - || (halfRight3 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0)) - || (halfRight2 != 0 && (halfLeft4 != 0 || halfLeft3 != 0)) - || (halfRight1 != 0 && halfLeft4 != 0)) { - return false; - } - - result[0] = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0; - result[1] = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2; - result[2] = product; - - return true; - } - - /** - * Fast decimal multiplication on two decimals whose results are permitted to go beyond - * 38 digits to the maximum possible 76 digits. The caller is responsible for scaling and - * rounding the results back to 38 or fewer digits. - * - * The caller is responsible for determining the signum. - * - * @param left0 - * @param left1 - * @param left2 - * @param right0 - * @param right1 - * @param right2 - * @param result This full result has 5 longs. - * @return Returns false if the multiplication resulted in an overflow. Values in result are - * undefined in that case. - */ - public static boolean fastMultiplyFullInternal( - long left0, long left1, long left2, - long right0, long right1, long right2, - long[] result) { - assert (result.length == 5); - if (result.length != 5) { - throw new IllegalArgumentException("Expecting result array length = 5"); - } - - long product; - - final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL; - final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL; - final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL; - final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL; - final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL; - - final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL; - final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL; - final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL; - final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL; - final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL; - - // v[0] - product = - halfRight0 * halfLeft0; - final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0]. - product = - halfRight0 - * halfLeft1 - + halfRight1 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[2] - product = - halfRight0 - * halfLeft2 - + halfRight1 - * halfLeft1 - + halfRight2 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[3] - product = - halfRight0 - * halfLeft3 - + halfRight1 - * halfLeft2 - + halfRight2 - * halfLeft1 - + halfRight3 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[4] - product = - halfRight0 - * halfLeft4 - + halfRight1 - * halfLeft3 - + halfRight2 - * halfLeft2 - + halfRight3 - * halfLeft1 - + halfRight4 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z4 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[5] -- since integer #5 is always 0, some products here are not included. - product = - halfRight1 - * halfLeft4 - + halfRight2 - * halfLeft3 - + halfRight3 - * halfLeft2 - + halfRight4 - * halfLeft1 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z5 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[6] -- since integer #5 is always 0, some products here are not included. - product = - halfRight2 - * halfLeft4 - + halfRight3 - * halfLeft3 - + halfRight4 - * halfLeft2 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z6 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[7] -- since integer #5 is always 0, some products here are not included. - product = - halfRight3 - * halfLeft4 - + halfRight4 - * halfLeft3 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z7 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[8] -- since integer #5 is always 0, some products here are not included. - product = - halfRight4 - * halfLeft4 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z8 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[9] -- since integer #5 is always 0, some products here are not included. - product = - (product / MULTIPLER_INTWORD_DECIMAL); - if (product > FULL_MAX_HIGHWORD_DECIMAL) { - return false; - } - - result[0] = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0; - result[1] = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2; - result[2] = (long) z5 * MULTIPLER_INTWORD_DECIMAL + (long) z4; - result[3] = (long) z7 * MULTIPLER_INTWORD_DECIMAL + (long) z6; - result[4] = product * MULTIPLER_INTWORD_DECIMAL + (long) z8; - - return true; - } - - /** - * Fast decimal multiplication on two decimals whose results are permitted to go beyond - * 38 digits to the maximum possible 76 digits. The caller is responsible for scaling and - * rounding the results back to 38 or fewer digits. - * - * The caller is responsible for determining the signum. - * - * @param result This full result has 5 longs. - * @return Returns false if the multiplication resulted in an overflow. Values in result are - * undefined in that case. - */ - public static boolean fastMultiply5x6HalfWords( - long left0, long left1, long left2, - long right0, long right1, long right2, - long[] result) { - - if (result.length != 6) { - throw new RuntimeException("Expecting result array length = 6"); - } - - long product; - - final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL; - final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL; - final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL; - final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL; - final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL; - final long halfRight5 = right2 / MULTIPLER_INTWORD_DECIMAL; - - final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL; - final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL; - final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL; - final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL; - final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL; - - // v[0] - product = - halfRight0 * halfLeft0; - final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0]. - product = - halfRight0 - * halfLeft1 - + halfRight1 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[2] - product = - halfRight0 - * halfLeft2 - + halfRight1 - * halfLeft1 - + halfRight2 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[3] - product = - halfRight0 - * halfLeft3 - + halfRight1 - * halfLeft2 - + halfRight2 - * halfLeft1 - + halfRight3 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[4] - product = - halfRight0 - * halfLeft4 - + halfRight1 - * halfLeft3 - + halfRight2 - * halfLeft2 - + halfRight3 - * halfLeft1 - + halfRight4 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z4 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[5] -- since left integer #5 is always 0, some products here are not included. - product = - halfRight1 - * halfLeft4 - + halfRight2 - * halfLeft3 - + halfRight3 - * halfLeft2 - + halfRight4 - * halfLeft1 - + halfRight5 - * halfLeft0 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z5 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[6] -- since left integer #5 is always 0, some products here are not included. - product = - halfRight2 - * halfLeft4 - + halfRight3 - * halfLeft3 - + halfRight4 - * halfLeft2 - + halfRight5 - * halfLeft1 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z6 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[7] -- since left integer #5 is always 0, some products here are not included. - product = - halfRight3 - * halfLeft4 - + halfRight4 - * halfLeft3 - + halfRight5 - * halfLeft2 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z7 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[8] -- since left integer #5 is always 0, some products here are not included. - product = - halfRight4 - * halfLeft4 - + halfRight5 - * halfLeft3 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z8 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[9] -- since left integer #5 is always 0, some products here are not included. - product = - halfRight5 - * halfLeft4 - + (product / MULTIPLER_INTWORD_DECIMAL); - final int z9 = (int) (product % MULTIPLER_INTWORD_DECIMAL); - - // v[10] -- since left integer #5 is always 0, some products here are not included. - product = - + (product / MULTIPLER_INTWORD_DECIMAL); - if (product > MULTIPLER_INTWORD_DECIMAL) { - return false; - } - - result[0] = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0; - result[1] = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2; - result[2] = (long) z5 * MULTIPLER_INTWORD_DECIMAL + (long) z4; - result[3] = (long) z7 * MULTIPLER_INTWORD_DECIMAL + (long) z6; - result[4] = (long) z9 * MULTIPLER_INTWORD_DECIMAL + (long) z8; - result[5] = product; - - return true; - } - - public static boolean fastMultiply( - FastHiveDecimal fastLeft, - FastHiveDecimal fastRight, - FastHiveDecimal fastResult) { - return fastMultiply( - fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, - fastLeft.fastIntegerDigitCount, fastLeft.fastScale, - fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, - fastRight.fastIntegerDigitCount, fastRight.fastScale, - fastResult); - } - - public static boolean fastMultiply( - int leftSignum, long leftFast0, long leftFast1, long leftFast2, - int leftIntegerDigitCount, int leftScale, - int rightSignum, long rightFast0, long rightFast1, long rightFast2, - int rightIntegerDigitCount, int rightScale, - FastHiveDecimal fastResult) { - - if (leftSignum == 0 || rightSignum == 0) { - fastResult.fastReset(); - return true; - } - - return doMultiply( - leftSignum, leftFast0, leftFast1, leftFast2, - leftIntegerDigitCount, leftScale, - rightSignum, rightFast0, rightFast1, rightFast2, - rightIntegerDigitCount, rightScale, - fastResult); - } - - //************************************************************************************************ - // Decimal Division / Remainder. - - /** - * EXPERMIMENTAL: Division when divisor fits in a single decimal longword. - * - * @return remainderSubexpr2 - */ - private static long doSingleWordQuotient( - long leftFast0, long leftFast1, long leftFast2, - long rightFast0, - FastHiveDecimal fastResult) { - - long quotient2; - long quotient1; - long quotient0; - - long remainderSubexpr2; - - if (leftFast2 == 0 && leftFast1 == 0) { - quotient2 = 0; - quotient1 = 0; - quotient0 = - leftFast0 / rightFast0; - final long k0 = - leftFast0 - quotient0 * rightFast0; - remainderSubexpr2 = - k0 * MULTIPLER_LONGWORD_DECIMAL; - } else if (leftFast2 == 0) { - // leftFast1 != 0. - quotient2 = 0; - quotient1 = - leftFast1 / rightFast0; - final long k1 = - leftFast1 - quotient1 * rightFast0; - final long quotientSubexpr0 = - k1 * MULTIPLER_LONGWORD_DECIMAL - + leftFast0; - quotient0 = - quotientSubexpr0 / rightFast0; - final long k0 = - quotientSubexpr0 - quotient0 * rightFast0; - remainderSubexpr2 = - k0 * MULTIPLER_LONGWORD_DECIMAL; - } else if (leftFast1 == 0){ - // leftFast2 != 0 && leftFast1 == 0. - quotient2 = - leftFast2 / rightFast0; - quotient1 = 0; - quotient0 = - leftFast0 / rightFast0; - final long k0 = - leftFast0 - quotient0 * rightFast0; - remainderSubexpr2 = - k0 * MULTIPLER_LONGWORD_DECIMAL; - } else { - quotient2 = - leftFast2 / rightFast0; - final long k2 = - leftFast2 - quotient2 * rightFast0; - final long quotientSubexpr1 = - k2 * MULTIPLER_LONGWORD_DECIMAL - + leftFast1; - quotient1 = - quotientSubexpr1 / rightFast0; - final long k1 = - quotientSubexpr1 - quotient1 * rightFast0; - final long quotientSubexpr0 = - k1 * MULTIPLER_LONGWORD_DECIMAL; - quotient0 = - quotientSubexpr0 / rightFast0; - final long k0 = - quotientSubexpr0 - quotient0 * rightFast0; - remainderSubexpr2 = - k0 * MULTIPLER_LONGWORD_DECIMAL; - } - - fastResult.fast0 = quotient0; - fastResult.fast1 = quotient1; - fastResult.fast2 = quotient2; - - return remainderSubexpr2; - } - - private static int doSingleWordRemainder( - long leftFast0, long leftFast1, long leftFast2, - long rightFast0, - long remainderSubexpr2, - FastHiveDecimal fastResult) { - - int remainderDigitCount; - - long remainder2; - long remainder1; - long remainder0; - - if (remainderSubexpr2 == 0) { - remainder2 = 0; - remainder1 = 0; - remainder0 = 0; - remainderDigitCount = 0; - } else { - remainder2 = - remainderSubexpr2 / rightFast0; - final long k2 = - remainderSubexpr2 - remainder2 * rightFast0; - if (k2 == 0) { - remainder1 = 0; - remainder0 = 0; - remainderDigitCount = - LONGWORD_DECIMAL_DIGITS - fastLongWordTrailingZeroCount(remainder2); - } else { - final long remainderSubexpr1 = - k2 * MULTIPLER_LONGWORD_DECIMAL; - long remainderSubexpr0; - remainder1 = - remainderSubexpr1 / rightFast0; - final long k1 = - remainderSubexpr1 - remainder1 * rightFast0; - if (k1 == 0) { - remainder0 = 0; - remainderDigitCount = - LONGWORD_DECIMAL_DIGITS - + LONGWORD_DECIMAL_DIGITS - fastLongWordTrailingZeroCount(remainder1); - } else { - remainderSubexpr0 = - k2 * MULTIPLER_LONGWORD_DECIMAL; - - remainder0 = - remainderSubexpr0 / rightFast0; - remainderDigitCount = - TWO_X_LONGWORD_DECIMAL_DIGITS - + LONGWORD_DECIMAL_DIGITS - fastLongWordTrailingZeroCount(remainder0); - } - } - } - - fastResult.fast0 = remainder0; - fastResult.fast1 = remainder1; - fastResult.fast2 = remainder2; - - return remainderDigitCount; - } - - // EXPERIMENT - private static boolean fastSingleWordDivision( - int leftSignum, long leftFast0, long leftFast1, long leftFast2, int leftScale, - int rightSignum, long rightFast0, int rightScale, - FastHiveDecimal fastResult) { - - long remainderSubexpr2 = - doSingleWordQuotient( - leftFast0, leftFast1, leftFast2, - rightFast0, - fastResult); - - long quotient0 = fastResult.fast0; - long quotient1 = fastResult.fast1; - long quotient2 = fastResult.fast2; - - int quotientDigitCount; - if (quotient2 != 0) { - quotientDigitCount = fastLongWordPrecision(quotient2); - } else if (quotient1 != 0) { - quotientDigitCount = fastLongWordPrecision(quotient1); - } else { - quotientDigitCount = fastLongWordPrecision(quotient0); - } - - int remainderDigitCount = - doSingleWordRemainder( - leftFast0, leftFast1, leftFast2, - rightFast0, - remainderSubexpr2, - fastResult); - - long remainder0 = fastResult.fast0; - long remainder1 = fastResult.fast1; - long remainder2 = fastResult.fast2; - - fastResult.fast0 = quotient0; - fastResult.fast1 = quotient1; - fastResult.fast2 = quotient2; - - final int quotientScale = leftScale + rightScale; - - if (remainderDigitCount == 0) { - fastResult.fastScale = quotientScale; - } else { - int resultScale = quotientScale + remainderDigitCount; - - int adjustedQuotientDigitCount; - if (quotientScale > 0) { - adjustedQuotientDigitCount = Math.max(0, quotientDigitCount - quotientScale); - } else { - adjustedQuotientDigitCount = quotientDigitCount; - } - final int maxScale = HiveDecimal.MAX_SCALE - adjustedQuotientDigitCount; - - int scale = Math.min(resultScale, maxScale); - - int remainderScale; - remainderScale = Math.min(remainderDigitCount, maxScale - quotientScale); - if (remainderScale > 0) { - if (quotientDigitCount > 0) { - // Make room for remainder. - fastScaleUp( - fastResult, - remainderScale, - fastResult); - } - // Copy in remainder digits... which start at the top of remainder2. - if (remainderScale < LONGWORD_DECIMAL_DIGITS) { - final long remainderDivisor2 = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - remainderScale]; - fastResult.fast0 += (remainder2 / remainderDivisor2); - } else if (remainderScale == LONGWORD_DECIMAL_DIGITS) { - fastResult.fast0 = remainder2; - } else if (remainderScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { - final long remainderDivisor2 = powerOfTenTable[remainderScale - LONGWORD_DECIMAL_DIGITS]; - fastResult.fast1 += (remainder2 / remainderDivisor2); - fastResult.fast0 = remainder1; - } else if (remainderScale == TWO_X_LONGWORD_DECIMAL_DIGITS) { - fastResult.fast1 = remainder2; - fastResult.fast0 = remainder1; - } - } - - // UNDONE: Method is still under development. - fastResult.fastScale = scale; - - // UNDONE: Trim trailing zeroes... - } - - return true; - } - - public static boolean fastDivide( - FastHiveDecimal fastLeft, - FastHiveDecimal fastRight, - FastHiveDecimal fastResult) { - return fastDivide( - fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, - fastLeft.fastIntegerDigitCount, fastLeft.fastScale, - fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, - fastRight.fastIntegerDigitCount, fastRight.fastScale, - fastResult); - } - - public static boolean fastDivide( - int leftSignum, long leftFast0, long leftFast1, long leftFast2, - int leftIntegerDigitCount, int leftScale, - int rightSignum, long rightFast0, long rightFast1, long rightFast2, - int rightIntegerDigitCount, int rightScale, - FastHiveDecimal fastResult) { - - // Arithmetic operations reset the results. - fastResult.fastReset(); - - if (rightSignum == 0) { - // Division by 0. - return false; - } - if (leftSignum == 0) { - // Zero result. - return true; - } - - /* - if (rightFast1 == 0 && rightFast2 == 0) { - return fastSingleWordDivision( - leftSignum, leftFast0, leftFast1, leftFast2, leftScale, - rightSignum, rightFast0, rightScale, - fastResult); - } - */ - - BigDecimal denominator = - fastBigDecimalValue( - leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale); - BigDecimal divisor = - fastBigDecimalValue( - rightSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale); - BigDecimal quotient = - denominator.divide(divisor, HiveDecimal.MAX_SCALE, BigDecimal.ROUND_HALF_UP); - - if (!fastSetFromBigDecimal( - quotient, - true, - fastResult)) { - return false; - } - - return true; - } - - public static boolean fastRemainder( - int leftSignum, long leftFast0, long leftFast1, long leftFast2, - int leftIntegerDigitCount, int leftScale, - int rightSignum, long rightFast0, long rightFast1, long rightFast2, - int rightIntegerDigitCount, int rightScale, - FastHiveDecimal fastResult) { - - // Arithmetic operations reset the results. - fastResult.fastReset(); - - if (rightSignum == 0) { - // Division by 0. - return false; - } - if (leftSignum == 0) { - // Zero result. - return true; - } - - BigDecimal denominator = - fastBigDecimalValue( - leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale); - BigDecimal divisor = - fastBigDecimalValue( - rightSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale); - BigDecimal remainder = - denominator.remainder(divisor); - fastResult.fastReset(); - if (!fastSetFromBigDecimal( - remainder, - true, - fastResult)) { - return false; - } - - return true; - } - - public static boolean fastPow( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int exponent, - FastHiveDecimal fastResult) { - - // Arithmetic operations (re)set the results. - fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - - if (exponent < 0) { - // UNDONE: Currently, negative exponent is not supported. - return false; - } - - for (int e = 1; e < exponent; e++) { - if (!doMultiply( - fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, - fastResult.fastIntegerDigitCount, fastResult.fastScale, - fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, - fastResult.fastIntegerDigitCount, fastResult.fastScale, - fastResult)) { - return false; - } - } - return true; - } - - //************************************************************************************************ - // Decimal String Formatting. - - public static String fastToFormatString( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int formatScale) { - byte[] scratchBuffer = new byte[FAST_SCRATCH_BUFFER_LEN_TO_BYTES]; - final int index = - doFastToFormatBytes( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - formatScale, - scratchBuffer); - return - new String(scratchBuffer, index, FAST_SCRATCH_BUFFER_LEN_TO_BYTES - index); - } - - //************************************************************************************************ - // Decimal String Formatting. - - public static String fastToFormatString( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int formatScale, - byte[] scratchBuffer) { - final int index = - doFastToBytes( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, formatScale, - scratchBuffer); - return new String(scratchBuffer, index, scratchBuffer.length - index); - } - - public static int fastToFormatBytes( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int formatScale, - byte[] scratchBuffer) { - return - doFastToFormatBytes( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - formatScale, - scratchBuffer); - } - - public static int doFastToFormatBytes( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, - int formatScale, - byte[] scratchBuffer) { - - // NOTE: OldHiveDecimal.toFormatString returns decimal strings with more than > 38 digits! - - if (formatScale >= fastScale) { - return - doFastToBytes( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, formatScale, - scratchBuffer); - } else { - FastHiveDecimal fastTemp = new FastHiveDecimal(); - if (!fastRound( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, - formatScale, BigDecimal.ROUND_HALF_UP, - fastTemp)) { - return 0; - } - return - doFastToBytes( - fastTemp.fastSignum, fastTemp.fast0, fastTemp.fast1, fastTemp.fast2, - fastTemp.fastIntegerDigitCount, fastTemp.fastScale, formatScale, - scratchBuffer); - } - } - - public static String fastToString( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale) { - return doFastToString( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, fastTrailingZeroesScale); - } - - public static String fastToString( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale, - byte[] scratchBuffer) { - return doFastToString( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, fastTrailingZeroesScale, - scratchBuffer); - } - - public static String fastToDigitsOnlyString( - long fast0, long fast1, long fast2, - int fastIntegerDigitCount) { - byte[] scratchBuffer = new byte[FAST_SCRATCH_BUFFER_LEN_TO_BYTES]; - final int index = - doFastToDigitsOnlyBytes( - fast0, fast1, fast2, - fastIntegerDigitCount, - scratchBuffer); - return - new String(scratchBuffer, index, FAST_SCRATCH_BUFFER_LEN_TO_BYTES - index); - } - - public static int fastToBytes( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale, - byte[] scratchBuffer) { - return doFastToBytes( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, fastTrailingZeroesScale, - scratchBuffer); - } - - private static String doFastToString( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale) { - byte[] scratchBuffer = new byte[FAST_SCRATCH_BUFFER_LEN_TO_BYTES]; - final int index = - doFastToBytes( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, fastTrailingZeroesScale, - scratchBuffer); - return - new String( - scratchBuffer, index, FAST_SCRATCH_BUFFER_LEN_TO_BYTES - index); - } - - private static String doFastToString( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale, - byte[] scratchBuffer) { - final int index = - doFastToBytes( - fastSignum, fast0, fast1, fast2, - fastIntegerDigitCount, fastScale, fastTrailingZeroesScale, - scratchBuffer); - return new String(scratchBuffer, index, scratchBuffer.length - index); - } - - private static int doFastToBytes( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale, - byte[] scratchBuffer) { - - int index = scratchBuffer.length - 1; - - int trailingZeroCount = - (fastTrailingZeroesScale != -1 ? fastTrailingZeroesScale - fastScale : 0); - // Virtual trailing zeroes. - if (trailingZeroCount > 0) { - for (int i = 0; i < trailingZeroCount; i++) { - scratchBuffer[index--] = BYTE_DIGIT_ZERO; - } - } - - // Scale fractional digits, dot, integer digits. - - final int scale = fastScale; - - final boolean isZeroFast1AndFast2 = (fast1 == 0 && fast2 == 0); - final boolean isZeroFast2 = (fast2 == 0); - - int lowerLongwordScale = 0; - int middleLongwordScale = 0; - int highLongwordScale = 0; - long longWord = fast0; - if (scale > 0) { - - // Fraction digits from lower longword. - - lowerLongwordScale = Math.min(scale, LONGWORD_DECIMAL_DIGITS); - - for (int i = 0; i < lowerLongwordScale; i++) { - scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); - longWord /= 10; - } - if (lowerLongwordScale == LONGWORD_DECIMAL_DIGITS) { - longWord = fast1; - } - - if (scale > LONGWORD_DECIMAL_DIGITS) { - - // Fraction digits continue into middle longword. - - middleLongwordScale = Math.min(scale - LONGWORD_DECIMAL_DIGITS, LONGWORD_DECIMAL_DIGITS); - for (int i = 0; i < middleLongwordScale; i++) { - scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); - longWord /= 10; - } - if (middleLongwordScale == LONGWORD_DECIMAL_DIGITS) { - longWord = fast2; - } - - if (scale > TWO_X_LONGWORD_DECIMAL_DIGITS) { - - // Fraction digit continue into highest longword. - - highLongwordScale = scale - TWO_X_LONGWORD_DECIMAL_DIGITS; - for (int i = 0; i < highLongwordScale; i++) { - scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); - longWord /= 10; - } - } - } - scratchBuffer[index--] = BYTE_DOT; - } else if (trailingZeroCount > 0) { - scratchBuffer[index--] = BYTE_DOT; - } - - // Integer digits; stop on zeroes above. - - boolean atLeastOneIntegerDigit = false; - if (scale <= LONGWORD_DECIMAL_DIGITS) { - - // Handle remaining lower long word digits as integer digits. - - final int remainingLowerLongwordDigits = LONGWORD_DECIMAL_DIGITS - lowerLongwordScale; - for (int i = 0; i < remainingLowerLongwordDigits; i++) { - scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); - atLeastOneIntegerDigit = true; - longWord /= 10; - if (longWord == 0 && isZeroFast1AndFast2) { - // Suppress leading zeroes. - break; - } - } - if (isZeroFast1AndFast2) { - if (!atLeastOneIntegerDigit) { - scratchBuffer[index--] = BYTE_DIGIT_ZERO; - } - if (fastSignum == -1) { - scratchBuffer[index--] = BYTE_MINUS; - } - return index + 1; - } - longWord = fast1; - } - - if (scale <= TWO_X_LONGWORD_DECIMAL_DIGITS) { - - // Handle remaining middle long word digits. - - final int remainingMiddleLongwordDigits = LONGWORD_DECIMAL_DIGITS - middleLongwordScale; - - for (int i = 0; i < remainingMiddleLongwordDigits; i++) { - scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); - atLeastOneIntegerDigit = true; - longWord /= 10; - if (longWord == 0 && isZeroFast2) { - // Suppress leading zeroes. - break; - } - } - if (isZeroFast2) { - if (!atLeastOneIntegerDigit) { - scratchBuffer[index--] = BYTE_DIGIT_ZERO; - } - if (fastSignum == -1) { - scratchBuffer[index--] = BYTE_MINUS; - } - return index + 1; - } - longWord = fast2; - } - - final int remainingHighwordDigits = HIGHWORD_DECIMAL_DIGITS - highLongwordScale; - - for (int i = 0; i < remainingHighwordDigits; i++) { - scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); - atLeastOneIntegerDigit = true; - longWord /= 10; - if (longWord == 0) { - // Suppress leading zeroes. - break; - } - } - if (!atLeastOneIntegerDigit) { - scratchBuffer[index--] = BYTE_DIGIT_ZERO; - } - if (fastSignum == -1) { - scratchBuffer[index--] = BYTE_MINUS; - } - return index + 1; - } - - public static int fastToDigitsOnlyBytes( - long fast0, long fast1, long fast2, - int fastIntegerDigitCount, - byte[] scratchBuffer) { - return doFastToDigitsOnlyBytes( - fast0, fast1, fast2, - fastIntegerDigitCount, - scratchBuffer); - } - - private static int doFastToDigitsOnlyBytes( - long fast0, long fast1, long fast2, - int fastIntegerDigitCount, - byte[] scratchBuffer) { - - int index = scratchBuffer.length - 1; - - // Just digits. - - final boolean isZeroFast1AndFast2 = (fast1 == 0 && fast2 == 0); - final boolean isZeroFast2 = (fast2 == 0); - - boolean atLeastOneIntegerDigit = false; - long longWord = fast0; - for (int i = 0; i < LONGWORD_DECIMAL_DIGITS; i++) { - scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); - atLeastOneIntegerDigit = true; - longWord /= 10; - if (longWord == 0 && isZeroFast1AndFast2) { - // Suppress leading zeroes. - break; - } - } - if (isZeroFast1AndFast2) { - if (!atLeastOneIntegerDigit) { - scratchBuffer[index--] = BYTE_DIGIT_ZERO; - } - return index + 1; - } - - longWord = fast1; - - for (int i = 0; i < LONGWORD_DECIMAL_DIGITS; i++) { - scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); - atLeastOneIntegerDigit = true; - longWord /= 10; - if (longWord == 0 && isZeroFast2) { - // Suppress leading zeroes. - break; - } - } - if (isZeroFast2) { - if (!atLeastOneIntegerDigit) { - scratchBuffer[index--] = BYTE_DIGIT_ZERO; - } - return index + 1; - } - - longWord = fast2; - - for (int i = 0; i < HIGHWORD_DECIMAL_DIGITS; i++) { - scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); - atLeastOneIntegerDigit = true; - longWord /= 10; - if (longWord == 0) { - // Suppress leading zeroes. - break; - } - } - if (!atLeastOneIntegerDigit) { - scratchBuffer[index--] = BYTE_DIGIT_ZERO; - } - return index + 1; - } - - //************************************************************************************************ - // Decimal Validation. - - public static boolean fastIsValid(FastHiveDecimal fastDec) { - return fastIsValid( - fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2, - fastDec.fastIntegerDigitCount, fastDec.fastScale); - } - - public static boolean fastIsValid( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - boolean isValid; - if (fastSignum == 0) { - isValid = (fast0 == 0 && fast1 == 0 && fast2 == 0 && fastIntegerDigitCount == 0 && fastScale == 0); - if (!isValid) { - System.out.println("FAST_IS_VALID signum 0 but other fields not"); - } - } else { - isValid = ( - (fast0 >= 0 && fast0 <= MAX_LONGWORD_DECIMAL) && - (fast1 >= 0 && fast1 <= MAX_LONGWORD_DECIMAL) && - (fast2 >= 0 && fast2 <= MAX_HIGHWORD_DECIMAL)); - if (!isValid) { - System.out.println("FAST_IS_VALID fast0 .. fast2 out of range"); - } else { - if (fastScale < 0 || fastScale > HiveDecimal.MAX_SCALE) { - System.out.println("FAST_IS_VALID fastScale " + fastScale + " out of range"); - isValid = false; - } else if (fastIntegerDigitCount < 0 || fastIntegerDigitCount > HiveDecimal.MAX_PRECISION) { - System.out.println("FAST_IS_VALID fastIntegerDigitCount " + fastIntegerDigitCount + " out of range"); - isValid = false; - } else if (fastIntegerDigitCount + fastScale > HiveDecimal.MAX_PRECISION) { - System.out.println("FAST_IS_VALID exceeds max precision: fastIntegerDigitCount " + fastIntegerDigitCount + " and fastScale " + fastScale); - isValid = false; - } else { - // Verify integerDigitCount given fastScale. - final int rawPrecision = fastRawPrecision(fastSignum, fast0, fast1, fast2); - if (fastIntegerDigitCount > 0) { - if (rawPrecision != fastIntegerDigitCount + fastScale) { - System.out.println("FAST_IS_VALID integer case: rawPrecision " + rawPrecision + - " fastIntegerDigitCount " + fastIntegerDigitCount + - " fastScale " + fastScale); - isValid = false; - } - } else { - if (rawPrecision > fastScale) { - System.out.println("FAST_IS_VALID fraction only case: rawPrecision " + rawPrecision + - " fastIntegerDigitCount " + fastIntegerDigitCount + - " fastScale " + fastScale); - isValid = false; - } - } - if (isValid) { - final int trailingZeroCount = - fastTrailingDecimalZeroCount( - fast0, fast1, fast2, - fastIntegerDigitCount, fastScale); - if (trailingZeroCount != 0) { - System.out.println("FAST_IS_VALID exceeds max precision: trailingZeroCount != 0"); - isValid = false; - } - } - } - } - } - - if (!isValid) { - System.out.println("FAST_IS_VALID fast0 " + fast0); - System.out.println("FAST_IS_VALID fast1 " + fast1); - System.out.println("FAST_IS_VALID fast2 " + fast2); - System.out.println("FAST_IS_VALID fastIntegerDigitCount " + fastIntegerDigitCount); - System.out.println("FAST_IS_VALID fastScale " + fastScale); - } - return isValid; - } - - public static void fastRaiseInvalidException( - FastHiveDecimal fastResult) { - throw new RuntimeException( - "Invalid fast decimal " + - " fastSignum " + fastResult.fastSignum + " fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2 + - " fastIntegerDigitCount " + fastResult.fastIntegerDigitCount + " fastScale " + fastResult.fastScale + - " stack trace: " + getStackTraceAsSingleLine(Thread.currentThread().getStackTrace())); - } - - public static void fastRaiseInvalidException( - FastHiveDecimal fastResult, - String parameters) { - throw new RuntimeException( - "Parameters: " + parameters + " --> " + - "Invalid fast decimal " + - " fastSignum " + fastResult.fastSignum + " fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2 + - " fastIntegerDigitCount " + fastResult.fastIntegerDigitCount + " fastScale " + fastResult.fastScale + - " stack trace: " + getStackTraceAsSingleLine(Thread.currentThread().getStackTrace())); - } - - //************************************************************************************************ - // Decimal Debugging. - - static final int STACK_LENGTH_LIMIT = 20; - public static String getStackTraceAsSingleLine(StackTraceElement[] stackTrace) { - StringBuilder sb = new StringBuilder(); - sb.append("Stack trace: "); - int length = stackTrace.length; - boolean isTruncated = false; - if (length > STACK_LENGTH_LIMIT) { - length = STACK_LENGTH_LIMIT; - isTruncated = true; - } - for (int i = 0; i < length; i++) { - if (i > 0) { - sb.append(", "); - } - sb.append(stackTrace[i]); - } - if (isTruncated) { - sb.append(", ..."); - } - - return sb.toString(); - } - - public static String displayBytes(byte[] bytes, int start, int length) { - StringBuilder sb = new StringBuilder(); - for (int i = start; i < start + length; i++) { - sb.append(String.format("\\%03d", (int) (bytes[i] & 0xff))); - } - return sb.toString(); - } -} \ No newline at end of file diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java deleted file mode 100644 index 7beb7bf935e35f00636d20b01782ec80a951ba5d..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java +++ /dev/null @@ -1,1505 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -import java.util.Arrays; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.math.BigDecimal; -import java.math.BigInteger; - -/** - * HiveDecimal is a decimal data type with a maximum precision and scale. - *

- * It is the Hive DECIMAL data type. - *

- * The scale is the number of fractional decimal digits. The digits after the dot. It is limited - * to 38 (MAX_SCALE). - *

- * The precision is the integer (or whole-number) decimal digits plus fractional decimal digits. - * It is limited to a total of 38 digits (MAX_PRECISION). - *

- * Hive syntax for declaring DECIMAL has 3 forms: - *

- * {@code - * DECIMAL // Use the default precision/scale.} - *

- * {@code - * DECIMAL(precision) // Use the default scale.} - *

- * {@code - * DECIMAL(precision, scale)} - * } - *

- * The declared scale must be <= precision. - *

- * Use DECIMAL instead of DOUBLE when exact numeric accuracy is required. Not all decimal numbers - * (radix 10) are exactly representable in the binary (radix 2 based) floating point type DOUBLE and - * cause accuracy anomalies (i.e. wrong results). See the Internet for more details. - *

- * HiveDecimal is implemented as a classic Java immutable object. All operations on HiveDecimal - * that produce a different value will create a new HiveDecimal object. - *

- * Decimals are physically stored without any extra leading or trailing zeroes. The scale of - * a decimal is the number of non-trailing zero fractional digits. - *

- * Math operations on decimals typically cause the scale to change as a result of the math and - * from trailing fractional digit elimination. - *

- * Typically, Hive, when it wants to make sure a result decimal fits in the column decimal's - * precision/scale it calls enforcePrecisionScale. That method will scale down or trim off - * result fractional digits if necessary with rounding when the column has a smaller scale. - * And, it will also indicate overflow when the decimal has exceeded the column's maximum precision. - *

- * NOTE: When Hive gets ready to serialize a decimal into text or binary, it usually sometimes - * wants trailing fractional zeroes. See the special notes for toFormatString and - * bigIntegerBytesScaled for details. - *

- * ------------------------------------- Version 2 ------------------------------------------------ - *

- * This is the 2nd major version of HiveDecimal called V2. The previous version has been - * renamed to HiveDecimalV1 and is kept as a test and behavior reference. - *

- * For good performance we do not represent the decimal using a BigDecimal object like the previous - * version V1 did. Using Java objects to represent our decimal incurs too high a penalty - * for memory allocations and general logic. - *

- * The original V1 public methods and fields are annotated with @HiveDecimalVersionV1; new public - * methods and fields are annotated with @HiveDecimalVersionV2. - * - */ -public final class HiveDecimal extends FastHiveDecimal implements Comparable { - - /* - * IMPLEMENTATION NOTE: - * We implement HiveDecimal with the mutable FastHiveDecimal class. That class uses - * protected on all its methods so they will not be visible in the HiveDecimal class. - * - * So even if one casts to FastHiveDecimal, you shouldn't be able to violate the immutability - * of a HiveDecimal class. - */ - - @HiveDecimalVersionV1 - public static final int MAX_PRECISION = 38; - @HiveDecimalVersionV1 - public static final int MAX_SCALE = 38; - - /** - * Default precision/scale when user doesn't specify in the column metadata, such as - * decimal and decimal(8). - */ - @HiveDecimalVersionV1 - public static final int USER_DEFAULT_PRECISION = 10; - @HiveDecimalVersionV1 - public static final int USER_DEFAULT_SCALE = 0; - - /** - * Default precision/scale when system is not able to determine them, such as in case - * of a non-generic udf. - */ - @HiveDecimalVersionV1 - public static final int SYSTEM_DEFAULT_PRECISION = 38; - @HiveDecimalVersionV1 - public static final int SYSTEM_DEFAULT_SCALE = 18; - - /** - * Common values. - */ - @HiveDecimalVersionV1 - public static final HiveDecimal ZERO = HiveDecimal.create(0); - @HiveDecimalVersionV1 - public static final HiveDecimal ONE = HiveDecimal.create(1); - - /** - * ROUND_FLOOR: - *

- * Round towards negative infinity. - *

- * The Hive function is FLOOR. - *

- * Positive numbers: The round fraction is thrown away. - *

- * (Example here rounds at scale 0) - * Value FLOOR - * 0.3 0 - * 2 2 - * 2.1 2 - *

- * Negative numbers: If there is a round fraction, throw it away and subtract 1. - *

- * (Example here rounds at scale 0) - * Value FLOOR - * -0.3 -1 - * -2 -2 - * -2.1 -3 - */ - @HiveDecimalVersionV1 - public static final int ROUND_FLOOR = BigDecimal.ROUND_FLOOR; - - /** - * ROUND_CEILING: - *

- * Round towards positive infinity. - *

- * The Hive function is CEILING. - *

- * Positive numbers: If there is a round fraction, throw it away and add 1 - *

- * (Example here rounds at scale 0) - * Value CEILING - * 0.3 1 - * 2 2 - * 2.1 3 - *

- * Negative numbers: The round fraction is thrown away. - *

- * (Example here rounds at scale 0) - * Value CEILING - * -0.3 0 - * -2 -2 - * -2.1 -2 - */ - @HiveDecimalVersionV1 - public static final int ROUND_CEILING = BigDecimal.ROUND_CEILING; - - /** - * ROUND_HALF_UP: - *

- * Round towards "nearest neighbor" unless both neighbors are equidistant then round up. - *

- * The Hive function is ROUND. - *

- * For result, throw away round fraction. If the round fraction is >= 0.5, then add 1 when - * positive and subtract 1 when negative. So, the sign is irrelevant. - *

- * (Example here rounds at scale 0) - * Value ROUND Value ROUND - * 0.3 0 -0.3 0 - * 2 2 -2 -2 - * 2.1 2 -2.1 -2 - * 2.49 2 -2.49 -2 - * 2.5 3 -2.5 -3 - * - */ - @HiveDecimalVersionV1 - public static final int ROUND_HALF_UP = BigDecimal.ROUND_HALF_UP; - - /** - * ROUND_HALF_EVEN: - * Round towards the "nearest neighbor" unless both neighbors are equidistant, then round - * towards the even neighbor. - *

- * The Hive function is BROUND. - *

- * Known as Banker’s Rounding. - *

- * When you add values rounded with ROUND_HALF_UP you have a bias that grows as you add more - * numbers. Banker's Rounding is a way to minimize that bias. It rounds toward the nearest - * even number when the fraction is 0.5 exactly. In table below, notice that 2.5 goes DOWN to - * 2 (even) but 3.5 goes UP to 4 (even), etc. - *

- * So, the sign is irrelevant. - *

- * (Example here rounds at scale 0) - * Value BROUND Value BROUND - * 0.49 0 -0.49 0 - * 0.5 0 -0.5 0 - * 0.51 1 -0.51 -1 - * 1.5 2 -1.5 -2 - * 2.5 2 -2.5 -2 - * 2.51 3 -2.51 -3 - * 3.5 4 -3.5 -4 - * 4.5 4 -4.5 -4 - * 4.51 5 -4.51 -5 - * - */ - @HiveDecimalVersionV1 - public static final int ROUND_HALF_EVEN = BigDecimal.ROUND_HALF_EVEN; - - //----------------------------------------------------------------------------------------------- - // Constructors are marked private; use create methods. - //----------------------------------------------------------------------------------------------- - - private HiveDecimal() { - super(); - } - - private HiveDecimal(HiveDecimal dec) { - super(dec); - } - - private HiveDecimal(FastHiveDecimal fastDec) { - super(fastDec); - } - - private HiveDecimal(int fastSignum, FastHiveDecimal fastDec) { - super(fastSignum, fastDec); - } - - private HiveDecimal( - int fastSignum, long fast0, long fast1, long fast2, - int fastIntegerDigitCount, int fastScale) { - super(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); - } - - //----------------------------------------------------------------------------------------------- - // Create methods. - //----------------------------------------------------------------------------------------------- - - /** - * Create a HiveDecimal from a FastHiveDecimal object. Used by HiveDecimalWritable. - * @param fastDec the value to set - * @return new hive decimal - */ - @HiveDecimalVersionV2 - public static HiveDecimal createFromFast(FastHiveDecimal fastDec) { - return new HiveDecimal(fastDec); - } - - /** - * Create a HiveDecimal from BigDecimal object. - * - * A BigDecimal object has a decimal scale. - * - * We will have overflow if BigDecimal's integer part exceed MAX_PRECISION digits or - * 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1. - * - * When the BigDecimal value's precision exceeds MAX_PRECISION and there are fractional digits - * because of scale > 0, then lower digits are trimmed off with rounding to meet the - * MAX_PRECISION requirement. - * - * Also, BigDecimal supports negative scale -- which means multiplying the value by 10^abs(scale). - * And, BigDecimal allows for a non-zero scale for zero. We normalize that so zero always has - * scale 0. - * - * @param bigDecimal the value to set - * @return The HiveDecimal with the BigDecimal's value adjusted down to a maximum precision. - * Otherwise, null is returned for overflow. - */ - @HiveDecimalVersionV1 - public static HiveDecimal create(BigDecimal bigDecimal) { - return create(bigDecimal, true); - } - - /** - * Same as the above create method, except fractional digit rounding can be turned off. - * @param bigDecimal the value to set - * @param allowRounding True requires all of the bigDecimal value be converted to the decimal - * without loss of precision. - * @return - */ - @HiveDecimalVersionV1 - public static HiveDecimal create(BigDecimal bigDecimal, boolean allowRounding) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromBigDecimal( - bigDecimal, allowRounding)) { - return null; - } - return result; - } - - /** - * Creates a HiveDecimal from a BigInteger's value with a scale of 0. - * - * We will have overflow if BigInteger exceed MAX_PRECISION digits or - * 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1. - * - * @param bigInteger the value to set - * @return A HiveDecimal object with the exact BigInteger's value. - * Otherwise, null is returned on overflow. - */ - @HiveDecimalVersionV1 - public static HiveDecimal create(BigInteger bigInteger) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromBigInteger( - bigInteger)) { - return null; - } - return result; - } - - /** - * Creates a HiveDecimal from a BigInteger's value with a specified scale. - * - * We will have overflow if BigInteger exceed MAX_PRECISION digits or - * 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1. - * - * The resulting decimal will have fractional digits when the specified scale is greater than 0. - * - * When the BigInteger's value's precision exceeds MAX_PRECISION and there are fractional digits - * because of scale > 0, then lower digits are trimmed off with rounding to meet the - * MAX_PRECISION requirement. - * - * @param bigInteger the value to set - * @param scale the scale to set - * @return A HiveDecimal object with the BigInteger's value adjusted for scale. - * Otherwise, null is returned on overflow. - */ - @HiveDecimalVersionV1 - public static HiveDecimal create(BigInteger bigInteger, int scale) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromBigIntegerAndScale( - bigInteger, scale)) { - return null; - } - return result; - } - - /** - * Create a HiveDecimal by parsing a whole string. - * - * We support parsing a decimal with an exponent because the previous version - * (i.e. OldHiveDecimal) uses the BigDecimal parser and was able to. - * - * @param string the string to parse - * @return a new hive decimal - */ - @HiveDecimalVersionV1 - public static HiveDecimal create(String string) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromString( - string, true)) { - return null; - } - return result; - } - - /** - * Same as the method above, except blanks before and after are tolerated. - * @param string the string to parse - * @param trimBlanks True specifies leading and trailing blanks are to be ignored. - * @return a new hive decimal - */ - @HiveDecimalVersionV2 - public static HiveDecimal create(String string, boolean trimBlanks) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromString( - string, trimBlanks)) { - return null; - } - return result; - } - - /** - * Create a HiveDecimal by parsing the characters in a whole byte array. - * - * Same rules as create(String string) above. - * - */ - @HiveDecimalVersionV2 - public static HiveDecimal create(byte[] bytes) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromBytes( - bytes, 0, bytes.length, false)) { - return null; - } - return result; - } - - /** - * Same as the method above, except blanks before and after are tolerated. - * - */ - @HiveDecimalVersionV2 - public static HiveDecimal create(byte[] bytes, boolean trimBlanks) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromBytes( - bytes, 0, bytes.length, trimBlanks)) { - return null; - } - return result; - } - - /** - * This method takes in digits only UTF-8 characters, a sign flag, and a scale and returns - * a decimal. - */ - @HiveDecimalVersionV2 - public static HiveDecimal create(boolean isNegative, byte[] bytes, int scale) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromDigitsOnlyBytesAndScale( - isNegative, bytes, 0, bytes.length, scale)) { - return null; - } - if (isNegative) { - result.fastNegate(); - } - return result; - } - - @HiveDecimalVersionV2 - public static HiveDecimal create( - boolean isNegative, byte[] bytes, int offset, int length, int scale) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromDigitsOnlyBytesAndScale( - isNegative, bytes, offset, length, scale)) { - return null; - } - return result; - } - - /** - * Create a HiveDecimal by parsing the characters in a slice of a byte array. - * - * Same rules as create(String string) above. - * - */ - @HiveDecimalVersionV2 - public static HiveDecimal create(byte[] bytes, int offset, int length) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromBytes( - bytes, offset, length, false)) { - return null; - } - return result; - } - - /** - * Same as the method above, except blanks before and after are tolerated. - * - */ - @HiveDecimalVersionV2 - public static HiveDecimal create( - byte[] bytes, int offset, int length, boolean trimBlanks) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromBytes( - bytes, offset, length, trimBlanks)) { - return null; - } - return result; - } - - /** - * Create a HiveDecimal object from an int. - * - */ - @HiveDecimalVersionV1 - public static HiveDecimal create(int intValue) { - HiveDecimal result = new HiveDecimal(); - result.fastSetFromInt(intValue); - return result; - } - - /** - * Create a HiveDecimal object from a long. - * - */ - @HiveDecimalVersionV1 - public static HiveDecimal create(long longValue) { - HiveDecimal result = new HiveDecimal(); - result.fastSetFromLong(longValue); - return result; - } - - /** - * Create a HiveDecimal object from a long with a specified scale. - * - */ - @HiveDecimalVersionV2 - public static HiveDecimal create(long longValue, int scale) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromLongAndScale( - longValue, scale)) { - return null; - } - return result; - } - - /** - * Create a HiveDecimal object from a float. - *

- * This method is equivalent to HiveDecimal.create(Float.toString(floatValue)) - */ - @HiveDecimalVersionV2 - public static HiveDecimal create(float floatValue) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromFloat(floatValue)) { - return null; - } - return result; - } - - /** - * Create a HiveDecimal object from a double. - *

- * This method is equivalent to HiveDecimal.create(Double.toString(doubleValue)) - */ - @HiveDecimalVersionV2 - public static HiveDecimal create(double doubleValue) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromDouble(doubleValue)) { - return null; - } - return result; - } - - //----------------------------------------------------------------------------------------------- - // Serialization methods. - //----------------------------------------------------------------------------------------------- - - // The byte length of the scratch byte array that needs to be passed to serializationUtilsRead. - @HiveDecimalVersionV2 - public static final int SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ = - FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ; - - /** - * Deserialize data written in the format used by the SerializationUtils methods - * readBigInteger/writeBigInteger and create a decimal using the supplied scale. - *

- * ORC uses those SerializationUtils methods for its serialization. - *

- * A scratch bytes array is necessary to do the binary to decimal conversion for better - * performance. Pass a SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ byte array for scratchBytes. - *

- * @return The deserialized decimal or null if the conversion failed. - */ - @HiveDecimalVersionV2 - public static HiveDecimal serializationUtilsRead( - InputStream inputStream, int scale, - byte[] scratchBytes) - throws IOException { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSerializationUtilsRead( - inputStream, scale, - scratchBytes)) { - return null; - } - return result; - } - - /** - * Convert bytes in the format used by BigInteger's toByteArray format (and accepted by its - * constructor) into a decimal using the specified scale. - *

- * Our bigIntegerBytes methods create bytes in this format, too. - *

- * This method is designed for high performance and does not create an actual BigInteger during - * binary to decimal conversion. - * - */ - @HiveDecimalVersionV2 - public static HiveDecimal createFromBigIntegerBytesAndScale( - byte[] bytes, int scale) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromBigIntegerBytesAndScale( - bytes, 0, bytes.length, scale)) { - return null; - } - return result; - } - - @HiveDecimalVersionV2 - public static HiveDecimal createFromBigIntegerBytesAndScale( - byte[] bytes, int offset, int length, int scale) { - HiveDecimal result = new HiveDecimal(); - if (!result.fastSetFromBigIntegerBytesAndScale( - bytes, offset, length, scale)) { - return null; - } - return result; - } - - // The length of the long array that needs to be passed to serializationUtilsWrite. - @HiveDecimalVersionV2 - public static final int SCRATCH_LONGS_LEN = FAST_SCRATCH_LONGS_LEN; - - /** - * Serialize this decimal's BigInteger equivalent unscaled value using the format that the - * SerializationUtils methods readBigInteger/writeBigInteger use. - *

- * ORC uses those SerializationUtils methods for its serialization. - *

- * Scratch objects necessary to do the decimal to binary conversion without actually creating a - * BigInteger object are passed for better performance. - *

- * Allocate scratchLongs with SCRATCH_LONGS_LEN longs. - * - */ - @HiveDecimalVersionV2 - public boolean serializationUtilsWrite( - OutputStream outputStream, - long[] scratchLongs) - throws IOException { - return - fastSerializationUtilsWrite( - outputStream, - scratchLongs); - } - - // The length of the scratch byte array that needs to be passed to bigIntegerBytes, etc. - @HiveDecimalVersionV2 - public static final int SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES = - FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES; - - /** - * Return binary representation of this decimal's BigInteger equivalent unscaled value using - * the format that the BigInteger's toByteArray method returns (and the BigInteger constructor - * accepts). - *

- * Used by LazyBinary, Avro, and Parquet serialization. - *

- * Scratch objects necessary to do the decimal to binary conversion without actually creating a - * BigInteger object are passed for better performance. - *

- * Allocate scratchLongs with SCRATCH_LONGS_LEN longs. - * And, allocate buffer with SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes. - *

- * @param scratchLongs - * @param buffer - * @return The number of bytes used for the binary result in buffer. Otherwise, 0 if the - * conversion failed. - */ - @HiveDecimalVersionV2 - public int bigIntegerBytes( - long[] scratchLongs, byte[] buffer) { - return - fastBigIntegerBytes( - scratchLongs, buffer); - } - - @HiveDecimalVersionV2 - public byte[] bigIntegerBytes() { - long[] scratchLongs = new long[SCRATCH_LONGS_LEN]; - byte[] buffer = new byte[SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; - final int byteLength = - fastBigIntegerBytes( - scratchLongs, buffer); - return Arrays.copyOfRange(buffer, 0, byteLength); - } - - /** - * Convert decimal to BigInteger binary bytes with a serialize scale, similar to the formatScale - * for toFormatString. It adds trailing zeroes the (emulated) BigInteger toByteArray result - * when a serializeScale is greater than current scale. Or, rounds if scale is less than - * current scale. - *

- * Used by Avro and Parquet serialization. - *

- * This emulates the OldHiveDecimal setScale AND THEN OldHiveDecimal getInternalStorage() behavior. - * - */ - @HiveDecimalVersionV2 - public int bigIntegerBytesScaled( - int serializeScale, - long[] scratchLongs, byte[] buffer) { - return - fastBigIntegerBytesScaled( - serializeScale, - scratchLongs, buffer); - } - - @HiveDecimalVersionV2 - public byte[] bigIntegerBytesScaled(int serializeScale) { - long[] scratchLongs = new long[SCRATCH_LONGS_LEN]; - byte[] buffer = new byte[SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; - int byteLength = - fastBigIntegerBytesScaled( - serializeScale, - scratchLongs, buffer); - return Arrays.copyOfRange(buffer, 0, byteLength); - } - - //----------------------------------------------------------------------------------------------- - // Convert to string/UTF-8 ASCII bytes methods. - //----------------------------------------------------------------------------------------------- - - /** - * Return a string representation of the decimal. - *

- * It is the equivalent of calling bigDecimalValue().toPlainString -- it does not add exponent - * notation -- but is much faster. - *

- * NOTE: If setScale(int serializationScale) was used to create the decimal object, then trailing - * fractional digits will be added to display to the serializationScale. Or, the display may - * get rounded. See the comments for that method. - * - */ - @HiveDecimalVersionV1 - @Override - public String toString() { - if (fastSerializationScale() != -1) { - - // Use the serialization scale and format the string with trailing zeroes (or - // round the decimal) if necessary. - return - fastToFormatString(fastSerializationScale()); - } else { - return - fastToString(); - } - } - - @HiveDecimalVersionV2 - public String toString( - byte[] scratchBuffer) { - if (fastSerializationScale() != -1) { - - // Use the serialization scale and format the string with trailing zeroes (or - // round the decimal) if necessary. - return - fastToFormatString( - fastSerializationScale(), - scratchBuffer); - } else { - return - fastToString(scratchBuffer); - } - } - - /** - * Return a string representation of the decimal using the specified scale. - *

- * This method is designed to ALWAYS SUCCEED (unless the newScale parameter is out of range). - *

- * Is does the equivalent of a setScale(int newScale). So, more than 38 digits may be returned. - * See that method for more details on how this can happen. - *

- * @param formatScale The number of digits after the decimal point - * @return The scaled decimal representation string representation. - */ - @HiveDecimalVersionV1 - public String toFormatString(int formatScale) { - return - fastToFormatString( - formatScale); - } - - @HiveDecimalVersionV2 - public String toFormatString(int formatScale, byte[] scratchBuffer) { - return - fastToFormatString( - formatScale, - scratchBuffer); - } - - @HiveDecimalVersionV2 - public String toDigitsOnlyString() { - return - fastToDigitsOnlyString(); - } - - // The length of the scratch buffer that needs to be passed to toBytes, toFormatBytes, - // toDigitsOnlyBytes. - @HiveDecimalVersionV2 - public final static int SCRATCH_BUFFER_LEN_TO_BYTES = FAST_SCRATCH_BUFFER_LEN_TO_BYTES; - - /** - * Decimal to ASCII bytes conversion. - *

- * The scratch buffer will contain the result afterwards. It should be - * SCRATCH_BUFFER_LEN_TO_BYTES bytes long. - *

- * The result is produced at the end of the scratch buffer, so the return value is the byte - * index of the first byte. The byte slice is [byteIndex:SCRATCH_BUFFER_LEN_TO_BYTES-1]. - * - */ - @HiveDecimalVersionV2 - public int toBytes( - byte[] scratchBuffer) { - return - fastToBytes( - scratchBuffer); - } - - /** - * This is the serialization version of decimal to string conversion. - *

- * It adds trailing zeroes when the formatScale is greater than the current scale. Or, it - * does round if the formatScale is less than the current scale. - *

- * Note that you can get more than 38 (MAX_PRECISION) digits in the output with this method. - * - */ - @HiveDecimalVersionV2 - public int toFormatBytes( - int formatScale, - byte[] scratchBuffer) { - return - fastToFormatBytes( - formatScale, - scratchBuffer); - } - - /** - * Convert decimal to just the digits -- no dot. - *

- * Currently used by BinarySortable serialization. - *

- * A faster way to get just the digits than calling unscaledValue.toString().getBytes(). - * - */ - @HiveDecimalVersionV2 - public int toDigitsOnlyBytes( - byte[] scratchBuffer) { - return - fastToDigitsOnlyBytes( - scratchBuffer); - } - - //----------------------------------------------------------------------------------------------- - // Comparison methods. - //----------------------------------------------------------------------------------------------- - - @HiveDecimalVersionV1 - @Override - public int compareTo(HiveDecimal dec) { - return fastCompareTo(dec); - } - - /** - * Hash code based on (new) decimal representation. - *

- * Faster than hashCode(). - *

- * Used by map join and other Hive internal purposes where performance is important. - *

- * IMPORTANT: See comments for hashCode(), too. - */ - @HiveDecimalVersionV2 - public int newFasterHashCode() { - return fastNewFasterHashCode(); - } - - /** - * This is returns original hash code as returned by HiveDecimalV1. - *

- * We need this when the HiveDecimalV1 hash code has been exposed and and written or affected - * how data is written. - *

- * This method supports compatibility. - *

- * Examples: bucketing, Hive hash() function, and Hive statistics. - *

- * NOTE: It is necessary to create a BigDecimal object and use its hash code, so this method is - * slow. - */ - @HiveDecimalVersionV1 - @Override - public int hashCode() { - return fastHashCode(); - } - - /** - * Are two decimal content (values) equal? - *

- * @param obj The 2nd decimal. - * @return When obj is null or not class HiveDecimal, the return is false. - * Otherwise, returns true when the decimal values are exactly equal. - */ - @HiveDecimalVersionV1 - @Override - public boolean equals(Object obj) { - if (obj == null || obj.getClass() != getClass()) { - return false; - } - return fastEquals((HiveDecimal) obj); - } - - - //----------------------------------------------------------------------------------------------- - // Attribute methods. - //----------------------------------------------------------------------------------------------- - - /** - * Returns the scale of the decimal. Range 0 .. MAX_SCALE. - * - */ - @HiveDecimalVersionV1 - public int scale() { - return fastScale(); - } - - /** - * Returns the number of integer digits in the decimal. - *

- * When the integer portion is zero, this method returns 0. - * - */ - @HiveDecimalVersionV2 - public int integerDigitCount() { - return fastIntegerDigitCount(); - } - - /** - * Returns the number of digits (integer and fractional) in the number, which is equivalent - * to SQL decimal precision. - *

- * Note that this method is different from rawPrecision(), which returns the number of digits - * ignoring the scale. Note that rawPrecision returns 0 when the value is 0. - * - * Decimal precision rawPrecision - * 0 1 0 - * 1 1 1 - * -7 1 1 - * 0.1 1 1 - * 0.04 2 1 - * 0.00380 5 3 - * 104.0009 7 7 - *

- * If you just want the actual number of digits, use rawPrecision(). - * - */ - @HiveDecimalVersionV1 - public int precision() { - return fastSqlPrecision(); - } - - // See comments for sqlPrecision. - @HiveDecimalVersionV2 - public int rawPrecision() { - return fastRawPrecision(); - } - - /** - * Get the sign of the decimal. - *

- * @return 0 if the decimal is equal to 0, -1 if less than zero, and 1 if greater than 0 - */ - @HiveDecimalVersionV1 - public int signum() { - return fastSignum(); - } - - //----------------------------------------------------------------------------------------------- - // Value conversion methods. - //----------------------------------------------------------------------------------------------- - - /** - * Is the decimal value a byte? Range -128 to 127. - * Byte.MIN_VALUE Byte.MAX_VALUE - *

- * Emulates testing for no value corruption: - * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().byteValue())) - *

- * NOTE: Fractional digits are ignored in the test since byteValue() will - * remove them (round down). - *

- * @return True when byteValue() will return a correct byte. - */ - @HiveDecimalVersionV2 - public boolean isByte() { - return fastIsByte(); - } - - /** - * A byte variation of longValue() - *

- * This method will return a corrupted value unless isByte() is true. - */ - @HiveDecimalVersionV1 - public byte byteValue() { - return fastByteValueClip(); - } - - /** - * Is the decimal value a short? Range -32,768 to 32,767. - * Short.MIN_VALUE Short.MAX_VALUE - *

- * Emulates testing for no value corruption: - * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().shortValue())) - *

- * NOTE: Fractional digits are ignored in the test since shortValue() will - * remove them (round down). - *

- * @return True when shortValue() will return a correct short. - */ - @HiveDecimalVersionV2 - public boolean isShort() { - return fastIsShort(); - } - - /** - * A short variation of longValue(). - *

- * This method will return a corrupted value unless isShort() is true. - */ - @HiveDecimalVersionV1 - public short shortValue() { - return fastShortValueClip(); - } - - /** - * Is the decimal value a int? Range -2,147,483,648 to 2,147,483,647. - * Integer.MIN_VALUE Integer.MAX_VALUE - *

- * Emulates testing for no value corruption: - * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().intValue())) - *

- * NOTE: Fractional digits are ignored in the test since intValue() will - * remove them (round down). - *

- * @return True when intValue() will return a correct int. - */ - @HiveDecimalVersionV2 - public boolean isInt() { - return fastIsInt(); - } - - /** - * An int variation of longValue(). - *

- * This method will return a corrupted value unless isInt() is true. - */ - @HiveDecimalVersionV1 - public int intValue() { - return fastIntValueClip(); - } - - /** - * Is the decimal value a long? Range -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807. - * Long.MIN_VALUE Long.MAX_VALUE - *

- * Emulates testing for no value corruption: - * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().longValue())) - *

- * NOTE: Fractional digits are ignored in the test since longValue() will - * remove them (round down). - *

- * @return True when longValue() will return a correct long. - */ - @HiveDecimalVersionV2 - public boolean isLong() { - return fastIsLong(); - } - - /** - * Return the long value of a decimal. - *

- * This method will return a corrupted value unless isLong() is true. - */ - @HiveDecimalVersionV1 - public long longValue() { - return fastLongValueClip(); - } - - @HiveDecimalVersionV1 - public long longValueExact() { - if (!isLong()) { - throw new ArithmeticException(); - } - return fastLongValueClip(); - } - - /** - * Return a float representing the decimal. Due the limitations of float, some values will not - * be accurate. - * - */ - @HiveDecimalVersionV1 - public float floatValue() { - return fastFloatValue(); - } - - /** - * Return a double representing the decimal. Due the limitations of double, some values will not - * be accurate. - * - */ - @HiveDecimalVersionV1 - public double doubleValue() { - return fastDoubleValue(); - } - - /** - * Return a BigDecimal representing the decimal. The BigDecimal class is able to accurately - * represent the decimal. - * - * NOTE: We are not representing our decimal as BigDecimal now as OldHiveDecimal did, so this - * is now slower. - * - */ - @HiveDecimalVersionV1 - public BigDecimal bigDecimalValue() { - return fastBigDecimalValue(); - } - - /** - * Get a BigInteger representing the decimal's digits without a dot. - *

- * @return Returns a signed BigInteger. - */ - @HiveDecimalVersionV1 - public BigInteger unscaledValue() { - return fastBigIntegerValue(); - } - - /** - * Return a decimal with only the fractional digits. - *

- * Zero is returned when there are no fractional digits (i.e. scale is 0). - * - */ - @HiveDecimalVersionV2 - public HiveDecimal fractionPortion() { - HiveDecimal result = new HiveDecimal(); - result.fastFractionPortion(); - return result; - } - - /** - * Return a decimal with only the integer digits. - *

- * Any fractional digits are removed. E.g. 2.083 scale 3 returns as 2 scale 0. - * - */ - @HiveDecimalVersionV2 - public HiveDecimal integerPortion() { - HiveDecimal result = new HiveDecimal(); - result.fastIntegerPortion(); - return result; - } - - //----------------------------------------------------------------------------------------------- - // Math methods. - //----------------------------------------------------------------------------------------------- - - /** - * Add the current decimal and another decimal and return the result. - * - */ - @HiveDecimalVersionV1 - public HiveDecimal add(HiveDecimal dec) { - HiveDecimal result = new HiveDecimal(); - if (!fastAdd( - dec, - result)) { - return null; - } - return result; - } - - /** - * Subtract from the current decimal another decimal and return the result. - * - */ - @HiveDecimalVersionV1 - public HiveDecimal subtract(HiveDecimal dec) { - HiveDecimal result = new HiveDecimal(); - if (!fastSubtract( - dec, - result)) { - return null; - } - return result; - } - - /** - * Multiply two decimals. - *

- * NOTE: Overflow Determination for Multiply - *

- * OldDecimal.multiply performs the multiply with BigDecimal but DOES NOT ALLOW ROUNDING - * (i.e. no throwing away lower fractional digits). - *

- * CONSIDER: Allowing rounding. This would eliminate cases today where we return null for - * the multiplication result. - *

- * IMPLEMENTATION NOTE: HiveDecimalV1 code does this: - *

- * return create(bd.multiply(dec.bd), false); - */ - @HiveDecimalVersionV1 - public HiveDecimal multiply(HiveDecimal dec) { - HiveDecimal result = new HiveDecimal(); - if (!fastMultiply( - dec, - result)) { - return null; - } - return result; - } - - /** - * Multiplies a decimal by a power of 10. - *

- * The decimal 19350 scale 0 will return 193.5 scale 1 when power is -2 (negative). - *

- * The decimal 1.000923 scale 6 will return 10009.23 scale 2 when power is 4 (positive). - *

- * @param power - * @return Returns a HiveDecimal whose value is value * 10^power. - */ - @HiveDecimalVersionV1 - public HiveDecimal scaleByPowerOfTen(int power) { - if (power == 0 || fastSignum() == 0) { - // No change for multiply by 10^0 or value 0. - return this; - } - HiveDecimal result = new HiveDecimal(); - if (!fastScaleByPowerOfTen( - power, - result)) { - return null; - } - return result; - } - - /** - * Take the absolute value of a decimal. - *

- * @return When the decimal is negative, returns a new HiveDecimal with the positive value. - * Otherwise, returns the current 0 or positive value object; - */ - @HiveDecimalVersionV1 - public HiveDecimal abs() { - if (fastSignum() != -1) { - return this; - } - HiveDecimal result = new HiveDecimal(this); - result.fastAbs(); - return result; - } - - /** - * Reverse the sign of a decimal. - *

- * @return Returns a new decimal with the sign flipped. When the value is 0, the current - * object is returned. - */ - @HiveDecimalVersionV1 - public HiveDecimal negate() { - if (fastSignum() == 0) { - return this; - } - HiveDecimal result = new HiveDecimal(this); - result.fastNegate(); - return result; - } - - //----------------------------------------------------------------------------------------------- - // Rounding / setScale methods. - //----------------------------------------------------------------------------------------------- - - /** - * DEPRECATED for V2. - *

- * Create a decimal from another decimal whose only change is it is MARKED and will display / - * serialize with a specified scale that will add trailing zeroes (or round) if necessary. - *

- * After display / serialization, the MARKED object is typically thrown away. - *

- * A MARKED decimal ONLY affects these 2 methods since these were the only ways setScale was - * used in the old code. - *

- * toString - * unscaleValue - *

- * This method has been deprecated because has poor performance by creating a throw away object. - *

- * For setScale(scale).toString() use toFormatString(scale) instead. - * For setScale(scale).unscaledValue().toByteArray() use V2 bigIntegerBytesScaled(scale) instead. - *

- * For better performance, use the V2 form of toFormatString that takes a scratch buffer, - * or even better use toFormatBytes. - *

- * And, use the form of bigIntegerBytesScaled that takes scratch objects for better performance. - * - */ - @Deprecated - @HiveDecimalVersionV1 - public HiveDecimal setScale(int serializationScale) { - HiveDecimal result = new HiveDecimal(this); - result.fastSetSerializationScale(serializationScale); - return result; - } - - /** - * Do decimal rounding and return the result. - *

- * When the roundingPoint is 0 or positive, we round away lower fractional digits if the - * roundingPoint is less than current scale. In this case, we will round the result using the - * specified rounding mode. - *

- * When the roundingPoint is negative, the rounding will occur within the integer digits. Integer - * digits below the roundPoint will be cleared. If the rounding occurred, a one will be added - * just above the roundingPoint. Note this may cause overflow. - *

- * No effect when the roundingPoint equals the current scale. The current object is returned. - *

- * The name setScale is taken from BigDecimal.setScale -- a better name would have been round. - * - */ - @HiveDecimalVersionV1 - public HiveDecimal setScale( - int roundingPoint, int roundingMode) { - if (fastScale() == roundingPoint) { - // No change. - return this; - } - - // Even if we are just setting the scale when newScale is greater than the current scale, - // we need a new object to obey our immutable behavior. - HiveDecimal result = new HiveDecimal(); - if (!fastRound( - roundingPoint, roundingMode, - result)) { - return null; - } - return result; - } - - /** - * Return the result of decimal^exponent - *

- * CONSIDER: Currently, negative exponent is not supported. - * CONSIDER: Does anybody use this method? - * - */ - @HiveDecimalVersionV1 - public HiveDecimal pow(int exponent) { - HiveDecimal result = new HiveDecimal(this); - if (!fastPow( - exponent, result)) { - return null; - } - return result; - } - - /** - * Divides this decimal by another decimal and returns a new decimal with the result. - * - */ - @HiveDecimalVersionV1 - public HiveDecimal divide(HiveDecimal divisor) { - HiveDecimal result = new HiveDecimal(); - if (!fastDivide( - divisor, - result)) { - return null; - } - return result; - } - - /** - * Divides this decimal by another decimal and returns a new decimal with the remainder of the - * division. - *

- * value is (decimal % divisor) - *

- * The remainder is equivalent to BigDecimal: - * bigDecimalValue().subtract(bigDecimalValue().divideToIntegralValue(divisor).multiply(divisor)) - * - */ - @HiveDecimalVersionV1 - public HiveDecimal remainder(HiveDecimal divisor) { - HiveDecimal result = new HiveDecimal(); - if (!fastRemainder( - divisor, - result)) { - return null; - } - return result; - } - - //----------------------------------------------------------------------------------------------- - // Precision/scale enforcement methods. - //----------------------------------------------------------------------------------------------- - - /** - * Determine if a decimal fits within a specified maxPrecision and maxScale, and round - * off fractional digits if necessary to make the decimal fit. - *

- * The relationship between the enforcement maxPrecision and maxScale is restricted. The - * specified maxScale must be less than or equal to the maxPrecision. - *

- * Normally, decimals that result from creation operation, arithmetic operations, etc are - * "free range" up to MAX_PRECISION and MAX_SCALE. Each operation checks if the result decimal - * is beyond MAX_PRECISION and MAX_SCALE. If so the result decimal is rounded off using - * ROUND_HALF_UP. If the round digit is 5 or more, one is added to the lowest remaining digit. - * The round digit is the digit just below the round point. Result overflow can occur if a - * result decimal's integer portion exceeds MAX_PRECISION. - *

- * This method supports enforcing to a declared Hive DECIMAL's precision/scale. - * E.g. DECIMAL(10,4) - *

- * Here are the enforcement/rounding checks of this method: - *

- * 1) Maximum integer digits = maxPrecision - maxScale - *

- * If the decimal's integer digit count exceeds this, the decimal does not fit (overflow). - *

- * 2) If decimal's scale is greater than maxScale, then excess fractional digits are - * rounded off. When rounding increases the remaining decimal, it may exceed the - * limits and overflow. - *

- * @param dec - * @param maxPrecision - * @param maxScale - * @return The original decimal if no adjustment is necessary. - * A rounded off decimal if adjustment was necessary. - * Otherwise, null if the decimal doesn't fit within maxPrecision / maxScale or rounding - * caused a result that exceeds the specified limits or MAX_PRECISION integer digits. - */ - @HiveDecimalVersionV1 - public static HiveDecimal enforcePrecisionScale( - HiveDecimal dec, int maxPrecision, int maxScale) { - - if (maxPrecision < 1 || maxPrecision > MAX_PRECISION) { - throw new IllegalArgumentException(STRING_ENFORCE_PRECISION_OUT_OF_RANGE); - } - - if (maxScale < 0 || maxScale > HiveDecimal.MAX_SCALE) { - throw new IllegalArgumentException(STRING_ENFORCE_SCALE_OUT_OF_RANGE); - } - - if (maxPrecision < maxScale) { - throw new IllegalArgumentException(STRING_ENFORCE_SCALE_LESS_THAN_EQUAL_PRECISION); - } - - if (dec == null) { - return null; - } - - FastCheckPrecisionScaleStatus status = - dec.fastCheckPrecisionScale( - maxPrecision, maxScale); - switch (status) { - case NO_CHANGE: - return dec; - case OVERFLOW: - return null; - case UPDATE_SCALE_DOWN: - { - HiveDecimal result = new HiveDecimal(); - if (!dec.fastUpdatePrecisionScale( - maxPrecision, maxScale, status, - result)) { - return null; - } - return result; - } - default: - throw new RuntimeException("Unknown fast decimal check precision and scale status " + status); - } - } - - //----------------------------------------------------------------------------------------------- - // Validation methods. - //----------------------------------------------------------------------------------------------- - - /** - * Throws an exception if the current decimal value is invalid. - */ - @HiveDecimalVersionV2 - public void validate() { - if (!fastIsValid()) { - fastRaiseInvalidException(); - } - } -} \ No newline at end of file diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalV1.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalV1.java deleted file mode 100644 index f99ffee32763ea3e61859e4c52cda9af85ed73a2..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalV1.java +++ /dev/null @@ -1,386 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.math.RoundingMode; - -/** - * - * HiveDecimal. Simple wrapper for BigDecimal. Adds fixed max precision and non scientific string - * representation - * - */ -public final class HiveDecimalV1 implements Comparable { - @HiveDecimalVersionV1 - public static final int MAX_PRECISION = 38; - @HiveDecimalVersionV1 - public static final int MAX_SCALE = 38; - - /** - * Default precision/scale when user doesn't specify in the column metadata, such as - * decimal and decimal(8). - */ - @HiveDecimalVersionV1 - public static final int USER_DEFAULT_PRECISION = 10; - @HiveDecimalVersionV1 - public static final int USER_DEFAULT_SCALE = 0; - - /** - * Default precision/scale when system is not able to determine them, such as in case - * of a non-generic udf. - */ - @HiveDecimalVersionV1 - public static final int SYSTEM_DEFAULT_PRECISION = 38; - @HiveDecimalVersionV1 - public static final int SYSTEM_DEFAULT_SCALE = 18; - - @HiveDecimalVersionV1 - public static final HiveDecimalV1 ZERO = new HiveDecimalV1(BigDecimal.ZERO); - @HiveDecimalVersionV1 - public static final HiveDecimalV1 ONE = new HiveDecimalV1(BigDecimal.ONE); - - @HiveDecimalVersionV1 - public static final int ROUND_FLOOR = BigDecimal.ROUND_FLOOR; - @HiveDecimalVersionV1 - public static final int ROUND_CEILING = BigDecimal.ROUND_CEILING; - @HiveDecimalVersionV1 - public static final int ROUND_HALF_UP = BigDecimal.ROUND_HALF_UP; - @HiveDecimalVersionV1 - public static final int ROUND_HALF_EVEN = BigDecimal.ROUND_HALF_EVEN; - - private BigDecimal bd = BigDecimal.ZERO; - - private HiveDecimalV1(BigDecimal bd) { - this.bd = bd; - } - - @HiveDecimalVersionV1 - public static HiveDecimalV1 create(BigDecimal b) { - return create(b, true); - } - - @HiveDecimalVersionV1 - public static HiveDecimalV1 create(BigDecimal b, boolean allowRounding) { - BigDecimal bd = normalize(b, allowRounding); - return bd == null ? null : new HiveDecimalV1(bd); - } - - @HiveDecimalVersionV1 - public static HiveDecimalV1 create(BigInteger unscaled, int scale) { - BigDecimal bd = normalize(new BigDecimal(unscaled, scale), true); - return bd == null ? null : new HiveDecimalV1(bd); - } - - @HiveDecimalVersionV1 - public static HiveDecimalV1 create(String dec) { - BigDecimal bd; - try { - bd = new BigDecimal(dec.trim()); - } catch (NumberFormatException ex) { - return null; - } - bd = normalize(bd, true); - return bd == null ? null : new HiveDecimalV1(bd); - } - - @HiveDecimalVersionV1 - public static HiveDecimalV1 create(BigInteger bi) { - BigDecimal bd = normalize(new BigDecimal(bi), true); - return bd == null ? null : new HiveDecimalV1(bd); - } - - @HiveDecimalVersionV1 - public static HiveDecimalV1 create(int i) { - return new HiveDecimalV1(new BigDecimal(i)); - } - - @HiveDecimalVersionV1 - public static HiveDecimalV1 create(long l) { - return new HiveDecimalV1(new BigDecimal(l)); - } - - @HiveDecimalVersionV1 - @Override - public String toString() { - return bd.toPlainString(); - } - - /** - * Return a string representation of the number with the number of decimal digits as - * the given scale. Please note that this is different from toString(). - * @param scale the number of digits after the decimal point - * @return the string representation of exact number of decimal digits - */ - @HiveDecimalVersionV1 - public String toFormatString(int scale) { - return (bd.scale() == scale ? bd : - bd.setScale(scale, RoundingMode.HALF_UP)).toPlainString(); - } - - @HiveDecimalVersionV1 - public HiveDecimalV1 setScale(int i) { - return new HiveDecimalV1(bd.setScale(i, RoundingMode.HALF_UP)); - } - - @HiveDecimalVersionV1 - @Override - public int compareTo(HiveDecimalV1 dec) { - return bd.compareTo(dec.bd); - } - - @HiveDecimalVersionV1 - @Override - public int hashCode() { - return bd.hashCode(); - } - - @HiveDecimalVersionV1 - @Override - public boolean equals(Object obj) { - if (obj == null || obj.getClass() != getClass()) { - return false; - } - return bd.equals(((HiveDecimalV1) obj).bd); - } - - @HiveDecimalVersionV1 - public int scale() { - return bd.scale(); - } - - /** - * Returns the number of digits (integer and fractional) in the number, which is equivalent - * to SQL decimal precision. Note that this is different from BigDecimal.precision(), - * which returns the precision of the unscaled value (BigDecimal.valueOf(0.01).precision() = 1, - * whereas HiveDecimal.create("0.01").precision() = 2). - * If you want the BigDecimal precision, use HiveDecimal.bigDecimalValue().precision() - * @return - */ - @HiveDecimalVersionV1 - public int precision() { - int bdPrecision = bd.precision(); - int bdScale = bd.scale(); - - if (bdPrecision < bdScale) { - // This can happen for numbers less than 0.1 - // For 0.001234: bdPrecision=4, bdScale=6 - // In this case, we'll set the type to have the same precision as the scale. - return bdScale; - } - return bdPrecision; - } - - /** Note - this method will corrupt the value if it doesn't fit. */ - @HiveDecimalVersionV1 - public int intValue() { - return bd.intValue(); - } - - @HiveDecimalVersionV1 - public double doubleValue() { - return bd.doubleValue(); - } - - /** Note - this method will corrupt the value if it doesn't fit. */ - @HiveDecimalVersionV1 - public long longValue() { - return bd.longValue(); - } - - /** Note - this method will corrupt the value if it doesn't fit. */ - @HiveDecimalVersionV1 - public short shortValue() { - return bd.shortValue(); - } - - @HiveDecimalVersionV1 - public float floatValue() { - return bd.floatValue(); - } - - @HiveDecimalVersionV1 - public BigDecimal bigDecimalValue() { - return bd; - } - - @HiveDecimalVersionV1 - public byte byteValue() { - return bd.byteValue(); - } - - @HiveDecimalVersionV1 - public HiveDecimalV1 setScale(int adjustedScale, int rm) { - return create(bd.setScale(adjustedScale, rm)); - } - - @HiveDecimalVersionV1 - public HiveDecimalV1 subtract(HiveDecimalV1 dec) { - return create(bd.subtract(dec.bd)); - } - - @HiveDecimalVersionV1 - public HiveDecimalV1 multiply(HiveDecimalV1 dec) { - return create(bd.multiply(dec.bd), false); - } - - @HiveDecimalVersionV1 - public BigInteger unscaledValue() { - return bd.unscaledValue(); - } - - @HiveDecimalVersionV1 - public HiveDecimalV1 scaleByPowerOfTen(int n) { - return create(bd.scaleByPowerOfTen(n)); - } - - @HiveDecimalVersionV1 - public HiveDecimalV1 abs() { - return create(bd.abs()); - } - - @HiveDecimalVersionV1 - public HiveDecimalV1 negate() { - return create(bd.negate()); - } - - @HiveDecimalVersionV1 - public HiveDecimalV1 add(HiveDecimalV1 dec) { - return create(bd.add(dec.bd)); - } - - @HiveDecimalVersionV1 - public HiveDecimalV1 pow(int n) { - BigDecimal result = normalize(bd.pow(n), false); - return result == null ? null : new HiveDecimalV1(result); - } - - @HiveDecimalVersionV1 - public HiveDecimalV1 remainder(HiveDecimalV1 dec) { - return create(bd.remainder(dec.bd)); - } - - @HiveDecimalVersionV1 - public HiveDecimalV1 divide(HiveDecimalV1 dec) { - return create(bd.divide(dec.bd, MAX_SCALE, RoundingMode.HALF_UP), true); - } - - /** - * Get the sign of the underlying decimal. - * @return 0 if the decimal is equal to 0, -1 if less than zero, and 1 if greater than 0 - */ - @HiveDecimalVersionV1 - public int signum() { - return bd.signum(); - } - - private static BigDecimal trim(BigDecimal d) { - if (d.compareTo(BigDecimal.ZERO) == 0) { - // Special case for 0, because java doesn't strip zeros correctly on that number. - d = BigDecimal.ZERO; - } else { - d = d.stripTrailingZeros(); - if (d.scale() < 0) { - // no negative scale decimals - d = d.setScale(0); - } - } - return d; - } - - private static BigDecimal normalize(BigDecimal bd, boolean allowRounding) { - if (bd == null) { - return null; - } - - bd = trim(bd); - - int intDigits = bd.precision() - bd.scale(); - - if (intDigits > MAX_PRECISION) { - return null; - } - - int maxScale = Math.min(MAX_SCALE, Math.min(MAX_PRECISION - intDigits, bd.scale())); - if (bd.scale() > maxScale ) { - if (allowRounding) { - bd = bd.setScale(maxScale, RoundingMode.HALF_UP); - // Trimming is again necessary, because rounding may introduce new trailing 0's. - bd = trim(bd); - } else { - bd = null; - } - } - - return bd; - } - - private static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, int maxScale) { - if (bd == null) { - return null; - } - - /** - * Specially handling the case that bd=0, and we are converting it to a type where precision=scale, - * such as decimal(1, 1). - */ - if (bd.compareTo(BigDecimal.ZERO) == 0 && bd.scale() == 0 && maxPrecision == maxScale) { - return bd.setScale(maxScale); - } - - bd = trim(bd); - - if (bd.scale() > maxScale) { - bd = bd.setScale(maxScale, RoundingMode.HALF_UP); - } - - int maxIntDigits = maxPrecision - maxScale; - int intDigits = bd.precision() - bd.scale(); - if (intDigits > maxIntDigits) { - return null; - } - - return bd; - } - - @HiveDecimalVersionV1 - public static HiveDecimalV1 enforcePrecisionScale(HiveDecimalV1 dec, int maxPrecision, int maxScale) { - if (dec == null) { - return null; - } - - // Minor optimization, avoiding creating new objects. - if (dec.precision() - dec.scale() <= maxPrecision - maxScale && - dec.scale() <= maxScale) { - return dec; - } - - BigDecimal bd = enforcePrecisionScale(dec.bd, maxPrecision, maxScale); - if (bd == null) { - return null; - } - - return HiveDecimalV1.create(bd); - } - - @HiveDecimalVersionV1 - public long longValueExact() { - return bd.longValueExact(); - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV1.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV1.java deleted file mode 100644 index 82b769a15eb9fa884f44894e8555b7c80dbdfc50..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV1.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.type; - -import java.lang.annotation.Documented; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -/** - * Marks methods including static methods and fields as being part of version 1 HiveDecimal. - * - */ -@Documented -@Retention(RetentionPolicy.RUNTIME) -public @interface HiveDecimalVersionV1 { - -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV2.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV2.java deleted file mode 100644 index a47513ebacb9c16600e9e2e50c194d13c19d39c9..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV2.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.type; - -import java.lang.annotation.Documented; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -/** - * Marks methods including static methods and fields as being part of version 2 HiveDecimal. - * - */ -@Documented -@Retention(RetentionPolicy.RUNTIME) -public @interface HiveDecimalVersionV2 { - -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java deleted file mode 100644 index cb1306ee78ca99c00ef1a2ab8fcf439029f58dd4..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java +++ /dev/null @@ -1,251 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -import java.math.BigDecimal; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.commons.lang.builder.HashCodeBuilder; -import org.apache.hive.common.util.IntervalDayTimeUtils; - - -/** - * Day-time interval type representing an offset in days/hours/minutes/seconds, - * with nanosecond precision. - * 1 day = 24 hours = 1440 minutes = 86400 seconds - */ -public class HiveIntervalDayTime implements Comparable { - - // days/hours/minutes/seconds all represented as seconds - protected long totalSeconds; - protected int nanos; - - public HiveIntervalDayTime() { - } - - public HiveIntervalDayTime(int days, int hours, int minutes, int seconds, int nanos) { - set(days, hours, minutes, seconds, nanos); - } - - public HiveIntervalDayTime(long seconds, int nanos) { - set(seconds, nanos); - } - - public HiveIntervalDayTime(BigDecimal seconds) { - set(seconds); - } - - public HiveIntervalDayTime(HiveIntervalDayTime other) { - set(other.totalSeconds, other.nanos); - } - - public int getDays() { - return (int) TimeUnit.SECONDS.toDays(totalSeconds); - } - - public int getHours() { - return (int) (TimeUnit.SECONDS.toHours(totalSeconds) % TimeUnit.DAYS.toHours(1)); - } - - public int getMinutes() { - return (int) (TimeUnit.SECONDS.toMinutes(totalSeconds) % TimeUnit.HOURS.toMinutes(1)); - } - - public int getSeconds() { - return (int) (totalSeconds % TimeUnit.MINUTES.toSeconds(1)); - } - - public int getNanos() { - return nanos; - } - - /** - * Returns days/hours/minutes all converted into seconds. - * Nanos still need to be retrieved using getNanos() - * @return - */ - public long getTotalSeconds() { - return totalSeconds; - } - - /** - * - * @return double representation of the interval day time, accurate to nanoseconds - */ - public double getDouble() { - return totalSeconds + nanos / 1000000000; - } - - /** - * Ensures that the seconds and nanoseconds fields have consistent sign - */ - protected void normalizeSecondsAndNanos() { - if (totalSeconds > 0 && nanos < 0) { - --totalSeconds; - nanos += IntervalDayTimeUtils.NANOS_PER_SEC; - } else if (totalSeconds < 0 && nanos > 0) { - ++totalSeconds; - nanos -= IntervalDayTimeUtils.NANOS_PER_SEC; - } - } - - public void set(int days, int hours, int minutes, int seconds, int nanos) { - long totalSeconds = seconds; - totalSeconds += TimeUnit.DAYS.toSeconds(days); - totalSeconds += TimeUnit.HOURS.toSeconds(hours); - totalSeconds += TimeUnit.MINUTES.toSeconds(minutes); - totalSeconds += TimeUnit.NANOSECONDS.toSeconds(nanos); - nanos = nanos % IntervalDayTimeUtils.NANOS_PER_SEC; - - this.totalSeconds = totalSeconds; - this.nanos = nanos; - - normalizeSecondsAndNanos(); - } - - public void set(long seconds, int nanos) { - this.totalSeconds = seconds; - this.nanos = nanos; - normalizeSecondsAndNanos(); - } - - public void set(BigDecimal totalSecondsBd) { - long totalSeconds = totalSecondsBd.longValue(); - BigDecimal fractionalSecs = totalSecondsBd.remainder(BigDecimal.ONE); - int nanos = fractionalSecs.multiply(IntervalDayTimeUtils.NANOS_PER_SEC_BD).intValue(); - set(totalSeconds, nanos); - } - - public void set(HiveIntervalDayTime other) { - set(other.getTotalSeconds(), other.getNanos()); - } - - public HiveIntervalDayTime negate() { - return new HiveIntervalDayTime(-getTotalSeconds(), -getNanos()); - } - - @Override - public int compareTo(HiveIntervalDayTime other) { - long cmp = this.totalSeconds - other.totalSeconds; - if (cmp == 0) { - cmp = this.nanos - other.nanos; - } - if (cmp != 0) { - cmp = cmp > 0 ? 1 : -1; - } - return (int) cmp; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof HiveIntervalDayTime)) { - return false; - } - return 0 == compareTo((HiveIntervalDayTime) obj); - } - - /** - * Return a copy of this object. - */ - @Override - public Object clone() { - return new HiveIntervalDayTime(totalSeconds, nanos); - } - - @Override - public int hashCode() { - return new HashCodeBuilder().append(totalSeconds).append(nanos).toHashCode(); - } - - @Override - public String toString() { - // If normalize() was used, then day-hour-minute-second-nanos should have the same sign. - // This is currently working with that assumption. - boolean isNegative = (totalSeconds < 0 || nanos < 0); - String daySecondSignStr = isNegative ? "-" : ""; - - return String.format("%s%d %02d:%02d:%02d.%09d", - daySecondSignStr, Math.abs(getDays()), - Math.abs(getHours()), Math.abs(getMinutes()), - Math.abs(getSeconds()), Math.abs(getNanos())); - } - - public static HiveIntervalDayTime valueOf(String strVal) { - HiveIntervalDayTime result = null; - if (strVal == null) { - throw new IllegalArgumentException("Interval day-time string was null"); - } - Matcher patternMatcher = PATTERN_MATCHER.get(); - patternMatcher.reset(strVal); - if (patternMatcher.matches()) { - // Parse out the individual parts - try { - // Sign - whether interval is positive or negative - int sign = 1; - String field = patternMatcher.group(1); - if (field != null && field.equals("-")) { - sign = -1; - } - int days = sign * - IntervalDayTimeUtils.parseNumericValueWithRange("day", patternMatcher.group(2), - 0, Integer.MAX_VALUE); - byte hours = (byte) (sign * - IntervalDayTimeUtils.parseNumericValueWithRange("hour", patternMatcher.group(3), 0, 23)); - byte minutes = (byte) (sign * - IntervalDayTimeUtils.parseNumericValueWithRange("minute", patternMatcher.group(4), 0, 59)); - int seconds = 0; - int nanos = 0; - field = patternMatcher.group(5); - if (field != null) { - BigDecimal bdSeconds = new BigDecimal(field); - if (bdSeconds.compareTo(IntervalDayTimeUtils.MAX_INT_BD) > 0) { - throw new IllegalArgumentException("seconds value of " + bdSeconds + " too large"); - } - seconds = sign * bdSeconds.intValue(); - nanos = sign * bdSeconds.subtract(new BigDecimal(bdSeconds.toBigInteger())) - .multiply(IntervalDayTimeUtils.NANOS_PER_SEC_BD).intValue(); - } - - result = new HiveIntervalDayTime(days, hours, minutes, seconds, nanos); - } catch (Exception err) { - throw new IllegalArgumentException("Error parsing interval day-time string: " + strVal, err); - } - } else { - throw new IllegalArgumentException( - "Interval string does not match day-time format of 'd h:m:s.n': " + strVal); - } - - return result; - } - - // Simple pattern: D H:M:S.nnnnnnnnn - private final static String PARSE_PATTERN = - "([+|-])?(\\d+) (\\d+):(\\d+):((\\d+)(\\.(\\d+))?)"; - - private static final ThreadLocal PATTERN_MATCHER = new ThreadLocal() { - @Override - protected Matcher initialValue() { - return Pattern.compile(PARSE_PATTERN).matcher(""); - } - }; -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java deleted file mode 100644 index eeb3359b3f225ff0f701f6d8c6c8a6aae8ae522d..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java +++ /dev/null @@ -1,187 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -import java.sql.Date; -import java.sql.Timestamp; -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Random; -import java.util.concurrent.TimeUnit; - -public class RandomTypeUtil { - - public static String getRandString(Random r) { - return getRandString(r, null, r.nextInt(10)); - } - - public static String getRandString(Random r, String characters, int length) { - if (characters == null) { - characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - - } - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < length; i++) { - if (characters == null) { - sb.append((char) (r.nextInt(128))); - } else { - sb.append(characters.charAt(r.nextInt(characters.length()))); - } - } - return sb.toString(); - } - - public static byte[] getRandBinary(Random r, int len){ - byte[] bytes = new byte[len]; - for (int j = 0; j < len; j++){ - bytes[j] = Byte.valueOf((byte) r.nextInt()); - } - return bytes; - } - - private static final String DECIMAL_CHARS = "0123456789"; - - public static HiveDecimal getRandHiveDecimal(Random r) { - int precision; - int scale; - while (true) { - StringBuilder sb = new StringBuilder(); - precision = 1 + r.nextInt(18); - scale = 0 + r.nextInt(precision + 1); - - int integerDigits = precision - scale; - - if (r.nextBoolean()) { - sb.append("-"); - } - - if (integerDigits == 0) { - sb.append("0"); - } else { - sb.append(getRandString(r, DECIMAL_CHARS, integerDigits)); - } - if (scale != 0) { - sb.append("."); - sb.append(getRandString(r, DECIMAL_CHARS, scale)); - } - - return HiveDecimal.create(sb.toString()); - } - } - - public static Date getRandDate(Random r) { - String dateStr = String.format("%d-%02d-%02d", - Integer.valueOf(1800 + r.nextInt(500)), // year - Integer.valueOf(1 + r.nextInt(12)), // month - Integer.valueOf(1 + r.nextInt(28))); // day - Date dateVal = Date.valueOf(dateStr); - return dateVal; - } - - /** - * TIMESTAMP. - */ - - public static final long NANOSECONDS_PER_SECOND = TimeUnit.SECONDS.toNanos(1); - public static final long MILLISECONDS_PER_SECOND = TimeUnit.SECONDS.toMillis(1); - public static final long NANOSECONDS_PER_MILLISSECOND = TimeUnit.MILLISECONDS.toNanos(1); - - private static final ThreadLocal DATE_FORMAT = - new ThreadLocal() { - @Override - protected DateFormat initialValue() { - return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - } - }; - - // We've switched to Joda/Java Calendar which has a more limited time range.... - public static final int MIN_YEAR = 1900; - public static final int MAX_YEAR = 3000; - private static final long MIN_FOUR_DIGIT_YEAR_MILLIS = parseToMillis("1900-01-01 00:00:00"); - private static final long MAX_FOUR_DIGIT_YEAR_MILLIS = parseToMillis("3000-01-01 00:00:00"); - - private static long parseToMillis(String s) { - try { - return DATE_FORMAT.get().parse(s).getTime(); - } catch (ParseException ex) { - throw new RuntimeException(ex); - } - } - - public static Timestamp getRandTimestamp(Random r) { - return getRandTimestamp(r, MIN_YEAR, MAX_YEAR); - } - - public static Timestamp getRandTimestamp(Random r, int minYear, int maxYear) { - String optionalNanos = ""; - switch (r.nextInt(4)) { - case 0: - // No nanos. - break; - case 1: - optionalNanos = String.format(".%09d", - Integer.valueOf(r.nextInt((int) NANOSECONDS_PER_SECOND))); - break; - case 2: - // Limit to milliseconds only... - optionalNanos = String.format(".%09d", - Integer.valueOf(r.nextInt((int) MILLISECONDS_PER_SECOND)) * NANOSECONDS_PER_MILLISSECOND); - break; - case 3: - // Limit to below milliseconds only... - optionalNanos = String.format(".%09d", - Integer.valueOf(r.nextInt((int) NANOSECONDS_PER_MILLISSECOND))); - break; - } - String timestampStr = String.format("%04d-%02d-%02d %02d:%02d:%02d%s", - Integer.valueOf(minYear + r.nextInt(maxYear - minYear + 1)), // year - Integer.valueOf(1 + r.nextInt(12)), // month - Integer.valueOf(1 + r.nextInt(28)), // day - Integer.valueOf(0 + r.nextInt(24)), // hour - Integer.valueOf(0 + r.nextInt(60)), // minute - Integer.valueOf(0 + r.nextInt(60)), // second - optionalNanos); - Timestamp timestampVal; - try { - timestampVal = Timestamp.valueOf(timestampStr); - } catch (Exception e) { - System.err.println("Timestamp string " + timestampStr + " did not parse"); - throw e; - } - return timestampVal; - } - - public static long randomMillis(long minMillis, long maxMillis, Random rand) { - return minMillis + (long) ((maxMillis - minMillis) * rand.nextDouble()); - } - - public static long randomMillis(Random rand) { - return randomMillis(MIN_FOUR_DIGIT_YEAR_MILLIS, MAX_FOUR_DIGIT_YEAR_MILLIS, rand); - } - - public static int randomNanos(Random rand, int decimalDigits) { - // Only keep the most significant decimalDigits digits. - int nanos = rand.nextInt((int) NANOSECONDS_PER_SECOND); - return nanos - nanos % (int) Math.pow(10, 9 - decimalDigits); - } - - public static int randomNanos(Random rand) { - return randomNanos(rand, 9); - } -} \ No newline at end of file diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java deleted file mode 100644 index c4f19cf1df49bba789432aa40931ae27d6619f0e..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java +++ /dev/null @@ -1,486 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector; - - -/** - * This class supports string and binary data by value reference -- i.e. each field is - * explicitly present, as opposed to provided by a dictionary reference. - * In some cases, all the values will be in the same byte array to begin with, - * but this need not be the case. If each value is in a separate byte - * array to start with, or not all of the values are in the same original - * byte array, you can still assign data by reference into this column vector. - * This gives flexibility to use this in multiple situations. - *

- * When setting data by reference, the caller - * is responsible for allocating the byte arrays used to hold the data. - * You can also set data by value, as long as you call the initBuffer() method first. - * You can mix "by value" and "by reference" in the same column vector, - * though that use is probably not typical. - */ -public class BytesColumnVector extends ColumnVector { - public byte[][] vector; - public int[] start; // start offset of each field - - /* - * The length of each field. If the value repeats for every entry, then it is stored - * in vector[0] and isRepeating from the superclass is set to true. - */ - public int[] length; - - // A call to increaseBufferSpace() or ensureValPreallocated() will ensure that buffer[] points to - // a byte[] with sufficient space for the specified size. - private byte[] buffer; // optional buffer to use when actually copying in data - private int nextFree; // next free position in buffer - - // Hang onto a byte array for holding smaller byte values - private byte[] smallBuffer; - private int smallBufferNextFree; - - private int bufferAllocationCount; - - // Estimate that there will be 16 bytes per entry - static final int DEFAULT_BUFFER_SIZE = 16 * VectorizedRowBatch.DEFAULT_SIZE; - - // Proportion of extra space to provide when allocating more buffer space. - static final float EXTRA_SPACE_FACTOR = (float) 1.2; - - // Largest size allowed in smallBuffer - static final int MAX_SIZE_FOR_SMALL_BUFFER = 1024 * 1024; - - /** - * Use this constructor for normal operation. - * All column vectors should be the default size normally. - */ - public BytesColumnVector() { - this(VectorizedRowBatch.DEFAULT_SIZE); - } - - /** - * Don't call this constructor except for testing purposes. - * - * @param size number of elements in the column vector - */ - public BytesColumnVector(int size) { - super(Type.BYTES, size); - vector = new byte[size][]; - start = new int[size]; - length = new int[size]; - } - - /** - * Additional reset work for BytesColumnVector (releasing scratch bytes for by value strings). - */ - @Override - public void reset() { - super.reset(); - initBuffer(0); - } - - /** Set a field by reference. - * - * @param elementNum index within column vector to set - * @param sourceBuf container of source data - * @param start start byte position within source - * @param length length of source byte sequence - */ - public void setRef(int elementNum, byte[] sourceBuf, int start, int length) { - vector[elementNum] = sourceBuf; - this.start[elementNum] = start; - this.length[elementNum] = length; - } - - /** - * You must call initBuffer first before using setVal(). - * Provide the estimated number of bytes needed to hold - * a full column vector worth of byte string data. - * - * @param estimatedValueSize Estimated size of buffer space needed - */ - public void initBuffer(int estimatedValueSize) { - nextFree = 0; - smallBufferNextFree = 0; - - // if buffer is already allocated, keep using it, don't re-allocate - if (buffer != null) { - // Free up any previously allocated buffers that are referenced by vector - if (bufferAllocationCount > 0) { - for (int idx = 0; idx < vector.length; ++idx) { - vector[idx] = null; - } - buffer = smallBuffer; // In case last row was a large bytes value - } - } else { - // allocate a little extra space to limit need to re-allocate - int bufferSize = this.vector.length * (int)(estimatedValueSize * EXTRA_SPACE_FACTOR); - if (bufferSize < DEFAULT_BUFFER_SIZE) { - bufferSize = DEFAULT_BUFFER_SIZE; - } - buffer = new byte[bufferSize]; - smallBuffer = buffer; - } - bufferAllocationCount = 0; - } - - /** - * Initialize buffer to default size. - */ - public void initBuffer() { - initBuffer(0); - } - - /** - * @return amount of buffer space currently allocated - */ - public int bufferSize() { - if (buffer == null) { - return 0; - } - return buffer.length; - } - - /** - * Set a field by actually copying in to a local buffer. - * If you must actually copy data in to the array, use this method. - * DO NOT USE this method unless it's not practical to set data by reference with setRef(). - * Setting data by reference tends to run a lot faster than copying data in. - * - * @param elementNum index within column vector to set - * @param sourceBuf container of source data - * @param start start byte position within source - * @param length length of source byte sequence - */ - public void setVal(int elementNum, byte[] sourceBuf, int start, int length) { - if ((nextFree + length) > buffer.length) { - increaseBufferSpace(length); - } - System.arraycopy(sourceBuf, start, buffer, nextFree, length); - vector[elementNum] = buffer; - this.start[elementNum] = nextFree; - this.length[elementNum] = length; - nextFree += length; - } - - /** - * Set a field by actually copying in to a local buffer. - * If you must actually copy data in to the array, use this method. - * DO NOT USE this method unless it's not practical to set data by reference with setRef(). - * Setting data by reference tends to run a lot faster than copying data in. - * - * @param elementNum index within column vector to set - * @param sourceBuf container of source data - */ - public void setVal(int elementNum, byte[] sourceBuf) { - setVal(elementNum, sourceBuf, 0, sourceBuf.length); - } - - /** - * Preallocate space in the local buffer so the caller can fill in the value bytes themselves. - * - * Always use with getValPreallocatedBytes, getValPreallocatedStart, and setValPreallocated. - */ - public void ensureValPreallocated(int length) { - if ((nextFree + length) > buffer.length) { - increaseBufferSpace(length); - } - } - - public byte[] getValPreallocatedBytes() { - return buffer; - } - - public int getValPreallocatedStart() { - return nextFree; - } - - /** - * Set the length of the preallocated values bytes used. - * @param elementNum - * @param length - */ - public void setValPreallocated(int elementNum, int length) { - vector[elementNum] = buffer; - this.start[elementNum] = nextFree; - this.length[elementNum] = length; - nextFree += length; - } - - /** - * Set a field to the concatenation of two string values. Result data is copied - * into the internal buffer. - * - * @param elementNum index within column vector to set - * @param leftSourceBuf container of left argument - * @param leftStart start of left argument - * @param leftLen length of left argument - * @param rightSourceBuf container of right argument - * @param rightStart start of right argument - * @param rightLen length of right arugment - */ - public void setConcat(int elementNum, byte[] leftSourceBuf, int leftStart, int leftLen, - byte[] rightSourceBuf, int rightStart, int rightLen) { - int newLen = leftLen + rightLen; - if ((nextFree + newLen) > buffer.length) { - increaseBufferSpace(newLen); - } - vector[elementNum] = buffer; - this.start[elementNum] = nextFree; - this.length[elementNum] = newLen; - - System.arraycopy(leftSourceBuf, leftStart, buffer, nextFree, leftLen); - nextFree += leftLen; - System.arraycopy(rightSourceBuf, rightStart, buffer, nextFree, rightLen); - nextFree += rightLen; - } - - /** - * Increase buffer space enough to accommodate next element. - * This uses an exponential increase mechanism to rapidly - * increase buffer size to enough to hold all data. - * As batches get re-loaded, buffer space allocated will quickly - * stabilize. - * - * @param nextElemLength size of next element to be added - */ - public void increaseBufferSpace(int nextElemLength) { - // A call to increaseBufferSpace() or ensureValPreallocated() will ensure that buffer[] points to - // a byte[] with sufficient space for the specified size. - // This will either point to smallBuffer, or to a newly allocated byte array for larger values. - - if (nextElemLength > MAX_SIZE_FOR_SMALL_BUFFER) { - // Larger allocations will be special-cased and will not use the normal buffer. - // buffer/nextFree will be set to a newly allocated array just for the current row. - // The next row will require another call to increaseBufferSpace() since this new buffer should be used up. - byte[] newBuffer = new byte[nextElemLength]; - ++bufferAllocationCount; - // If the buffer was pointing to smallBuffer, then nextFree keeps track of the current state - // of the free index for smallBuffer. We now need to save this value to smallBufferNextFree - // so we don't lose this. A bit of a weird dance here. - if (smallBuffer == buffer) { - smallBufferNextFree = nextFree; - } - buffer = newBuffer; - nextFree = 0; - } else { - // This value should go into smallBuffer. - if (smallBuffer != buffer) { - // Previous row was for a large bytes value ( > MAX_SIZE_FOR_SMALL_BUFFER). - // Use smallBuffer if possible. - buffer = smallBuffer; - nextFree = smallBufferNextFree; - } - - // smallBuffer might still be out of space - if ((nextFree + nextElemLength) > buffer.length) { - int newLength = smallBuffer.length * 2; - while (newLength < nextElemLength) { - if (newLength < 0) { - throw new RuntimeException("Overflow of newLength. smallBuffer.length=" - + smallBuffer.length + ", nextElemLength=" + nextElemLength); - } - newLength *= 2; - } - smallBuffer = new byte[newLength]; - ++bufferAllocationCount; - smallBufferNextFree = 0; - // Update buffer - buffer = smallBuffer; - nextFree = 0; - } - } - } - - /** Copy the current object contents into the output. Only copy selected entries, - * as indicated by selectedInUse and the sel array. - */ - public void copySelected( - boolean selectedInUse, int[] sel, int size, BytesColumnVector output) { - - // Output has nulls if and only if input has nulls. - output.noNulls = noNulls; - output.isRepeating = false; - - // Handle repeating case - if (isRepeating) { - output.setVal(0, vector[0], start[0], length[0]); - output.isNull[0] = isNull[0]; - output.isRepeating = true; - return; - } - - // Handle normal case - - // Copy data values over - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - output.setVal(i, vector[i], start[i], length[i]); - } - } - else { - for (int i = 0; i < size; i++) { - output.setVal(i, vector[i], start[i], length[i]); - } - } - - // Copy nulls over if needed - if (!noNulls) { - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - output.isNull[i] = isNull[i]; - } - } - else { - System.arraycopy(isNull, 0, output.isNull, 0, size); - } - } - } - - /** Simplify vector by brute-force flattening noNulls and isRepeating - * This can be used to reduce combinatorial explosion of code paths in VectorExpressions - * with many arguments, at the expense of loss of some performance. - */ - public void flatten(boolean selectedInUse, int[] sel, int size) { - flattenPush(); - if (isRepeating) { - isRepeating = false; - - // setRef is used below and this is safe, because the reference - // is to data owned by this column vector. If this column vector - // gets re-used, the whole thing is re-used together so there - // is no danger of a dangling reference. - - // Only copy data values if entry is not null. The string value - // at position 0 is undefined if the position 0 value is null. - if (noNulls || !isNull[0]) { - - // loops start at position 1 because position 0 is already set - if (selectedInUse) { - for (int j = 1; j < size; j++) { - int i = sel[j]; - this.setRef(i, vector[0], start[0], length[0]); - } - } else { - for (int i = 1; i < size; i++) { - this.setRef(i, vector[0], start[0], length[0]); - } - } - } - flattenRepeatingNulls(selectedInUse, sel, size); - } - flattenNoNulls(selectedInUse, sel, size); - } - - // Fill the all the vector entries with provided value - public void fill(byte[] value) { - noNulls = true; - isRepeating = true; - setRef(0, value, 0, value.length); - } - - // Fill the column vector with nulls - public void fillWithNulls() { - noNulls = false; - isRepeating = true; - vector[0] = null; - isNull[0] = true; - } - - @Override - public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) { - if (inputVector.isRepeating) { - inputElementNum = 0; - } - if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) { - isNull[outElementNum] = false; - BytesColumnVector in = (BytesColumnVector) inputVector; - setVal(outElementNum, in.vector[inputElementNum], - in.start[inputElementNum], in.length[inputElementNum]); - } else { - isNull[outElementNum] = true; - noNulls = false; - } - } - - @Override - public void init() { - initBuffer(0); - } - - public String toString(int row) { - if (isRepeating) { - row = 0; - } - if (noNulls || !isNull[row]) { - return new String(vector[row], start[row], length[row]); - } else { - return null; - } - } - - @Override - public void stringifyValue(StringBuilder buffer, int row) { - if (isRepeating) { - row = 0; - } - if (noNulls || !isNull[row]) { - buffer.append('"'); - buffer.append(new String(vector[row], start[row], length[row])); - buffer.append('"'); - } else { - buffer.append("null"); - } - } - - @Override - public void ensureSize(int size, boolean preserveData) { - super.ensureSize(size, preserveData); - if (size > vector.length) { - int[] oldStart = start; - start = new int[size]; - int[] oldLength = length; - length = new int[size]; - byte[][] oldVector = vector; - vector = new byte[size][]; - if (preserveData) { - if (isRepeating) { - vector[0] = oldVector[0]; - start[0] = oldStart[0]; - length[0] = oldLength[0]; - } else { - System.arraycopy(oldVector, 0, vector, 0, oldVector.length); - System.arraycopy(oldStart, 0, start, 0 , oldStart.length); - System.arraycopy(oldLength, 0, length, 0, oldLength.length); - } - } - } - } - - @Override - public void shallowCopyTo(ColumnVector otherCv) { - BytesColumnVector other = (BytesColumnVector)otherCv; - super.shallowCopyTo(other); - other.nextFree = nextFree; - other.vector = vector; - other.start = start; - other.length = length; - other.buffer = buffer; - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java deleted file mode 100644 index 57342729fe2b1d2e20d3e51088d59ac3fe4fbd83..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java +++ /dev/null @@ -1,233 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector; - -import java.util.Arrays; - -/** - * ColumnVector contains the shared structure for the sub-types, - * including NULL information, and whether this vector - * repeats, i.e. has all values the same, so only the first - * one is set. This is used to accelerate query performance - * by handling a whole vector in O(1) time when applicable. - * - * The fields are public by design since this is a performance-critical - * structure that is used in the inner loop of query execution. - */ -public abstract class ColumnVector { - - /* - * The current kinds of column vectors. - */ - public static enum Type { - NONE, // Useful when the type of column vector has not be determined yet. - LONG, - DOUBLE, - BYTES, - DECIMAL, - DECIMAL_64, - TIMESTAMP, - INTERVAL_DAY_TIME, - STRUCT, - LIST, - MAP, - UNION - } - - public final Type type; - - /* - * If hasNulls is true, then this array contains true if the value - * is null, otherwise false. The array is always allocated, so a batch can be re-used - * later and nulls added. - */ - public boolean[] isNull; - - // If the whole column vector has no nulls, this is true, otherwise false. - public boolean noNulls; - - /* - * True if same value repeats for whole column vector. - * If so, vector[0] holds the repeating value. - */ - public boolean isRepeating; - - // Variables to hold state from before flattening so it can be easily restored. - private boolean preFlattenIsRepeating; - private boolean preFlattenNoNulls; - - /** - * Constructor for super-class ColumnVector. This is not called directly, - * but used to initialize inherited fields. - * - * @param len Vector length - */ - public ColumnVector(Type type, int len) { - this.type = type; - isNull = new boolean[len]; - noNulls = true; - isRepeating = false; - preFlattenNoNulls = true; - preFlattenIsRepeating = false; - } - - /** - * Resets the column to default state - * - fills the isNull array with false - * - sets noNulls to true - * - sets isRepeating to false - */ - public void reset() { - if (!noNulls) { - Arrays.fill(isNull, false); - } - noNulls = true; - isRepeating = false; - preFlattenNoNulls = true; - preFlattenIsRepeating = false; - } - - /** - * Sets the isRepeating flag. Recurses over structs and unions so that the - * flags are set correctly. - * @param isRepeating - */ - public void setRepeating(boolean isRepeating) { - this.isRepeating = isRepeating; - } - - abstract public void flatten(boolean selectedInUse, int[] sel, int size); - - // Simplify vector by brute-force flattening noNulls if isRepeating - // This can be used to reduce combinatorial explosion of code paths in VectorExpressions - // with many arguments. - protected void flattenRepeatingNulls(boolean selectedInUse, int[] sel, - int size) { - - boolean nullFillValue; - - if (noNulls) { - nullFillValue = false; - } else { - nullFillValue = isNull[0]; - } - - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - isNull[i] = nullFillValue; - } - } else { - Arrays.fill(isNull, 0, size, nullFillValue); - } - - // all nulls are now explicit - noNulls = false; - } - - protected void flattenNoNulls(boolean selectedInUse, int[] sel, - int size) { - if (noNulls) { - noNulls = false; - if (selectedInUse) { - for (int j = 0; j < size; j++) { - isNull[sel[j]] = false; - } - } else { - Arrays.fill(isNull, 0, size, false); - } - } - } - - /** - * Restore the state of isRepeating and noNulls to what it was - * before flattening. This must only be called just after flattening - * and then evaluating a VectorExpression on the column vector. - * It is an optimization that allows other operations on the same - * column to continue to benefit from the isRepeating and noNulls - * indicators. - */ - public void unFlatten() { - isRepeating = preFlattenIsRepeating; - noNulls = preFlattenNoNulls; - } - - // Record repeating and no nulls state to be restored later. - protected void flattenPush() { - preFlattenIsRepeating = isRepeating; - preFlattenNoNulls = noNulls; - } - - /** - * Set the element in this column vector from the given input vector. - * This method can assume that the output does not have isRepeating set. - */ - public abstract void setElement(int outElementNum, int inputElementNum, - ColumnVector inputVector); - - /** - * Initialize the column vector. This method can be overridden by specific column vector types. - * Use this method only if the individual type of the column vector is not known, otherwise its - * preferable to call specific initialization methods. - */ - public void init() { - // Do nothing by default - } - - /** - * Ensure the ColumnVector can hold at least size values. - * This method is deliberately *not* recursive because the complex types - * can easily have more (or less) children than the upper levels. - * @param size the new minimum size - * @param preserveData should the old data be preserved? - */ - public void ensureSize(int size, boolean preserveData) { - if (isNull.length < size) { - boolean[] oldArray = isNull; - isNull = new boolean[size]; - if (preserveData && !noNulls) { - if (isRepeating) { - isNull[0] = oldArray[0]; - } else { - System.arraycopy(oldArray, 0, isNull, 0, oldArray.length); - } - } - } - } - - /** - * Print the value for this column into the given string builder. - * @param buffer the buffer to print into - * @param row the id of the row to print - */ - public abstract void stringifyValue(StringBuilder buffer, - int row); - - /** - * Shallow copy of the contents of this vector to the other vector; - * replaces other vector's values. - */ - public void shallowCopyTo(ColumnVector otherCv) { - otherCv.isNull = isNull; - otherCv.noNulls = noNulls; - otherCv.isRepeating = isRepeating; - otherCv.preFlattenIsRepeating = preFlattenIsRepeating; - otherCv.preFlattenNoNulls = preFlattenNoNulls; - } - } diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java deleted file mode 100644 index 5548b9dfbfd08efac0119427165f2981fb4f2229..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.exec.vector; - -import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; - -/** - - */ -public class Decimal64ColumnVector extends LongColumnVector { - - public short scale; - public short precision; - - private HiveDecimalWritable tempHiveDecWritable; - - public Decimal64ColumnVector(int precision, int scale) { - this(VectorizedRowBatch.DEFAULT_SIZE, precision, scale); - } - - public Decimal64ColumnVector(int size, int precision, int scale) { - super(size); - this.precision = (short) precision; - this.scale = (short) scale; - tempHiveDecWritable = new HiveDecimalWritable(); - } - - public void set(int elementNum, HiveDecimalWritable writable) { - tempHiveDecWritable.set(writable); - tempHiveDecWritable.mutateEnforcePrecisionScale(precision, scale); - if (!tempHiveDecWritable.isSet()) { - noNulls = false; - isNull[elementNum] = true; - } else { - isNull[elementNum] = false; - vector[elementNum] = tempHiveDecWritable.serialize64(scale); - } - } - - public void set(int elementNum, HiveDecimal hiveDec) { - tempHiveDecWritable.set(hiveDec); - tempHiveDecWritable.mutateEnforcePrecisionScale(precision, scale); - if (!tempHiveDecWritable.isSet()) { - noNulls = false; - isNull[elementNum] = true; - } else { - isNull[elementNum] = false; - vector[elementNum] = tempHiveDecWritable.serialize64(scale); - } - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java deleted file mode 100644 index cfe40aca4908701b8d71955f0d52b3cb1f1a72cd..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java +++ /dev/null @@ -1,152 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector; - - -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; -import org.apache.hadoop.hive.common.type.HiveDecimal; - -public class DecimalColumnVector extends ColumnVector { - - /** - * A vector of HiveDecimalWritable objects. - * - * For high performance and easy access to this low-level structure, - * the fields are public by design (as they are in other ColumnVector - * types). - */ - public HiveDecimalWritable[] vector; - public short scale; - public short precision; - - public DecimalColumnVector(int precision, int scale) { - this(VectorizedRowBatch.DEFAULT_SIZE, precision, scale); - } - - public DecimalColumnVector(int size, int precision, int scale) { - super(Type.DECIMAL, size); - this.precision = (short) precision; - this.scale = (short) scale; - vector = new HiveDecimalWritable[size]; - for (int i = 0; i < size; i++) { - vector[i] = new HiveDecimalWritable(0); // Initially zero. - } - } - - // Fill the all the vector entries with provided value - public void fill(HiveDecimal value) { - noNulls = true; - isRepeating = true; - if (vector[0] == null) { - vector[0] = new HiveDecimalWritable(value); - } else { - vector[0].set(value); - } - } - - @Override - public void flatten(boolean selectedInUse, int[] sel, int size) { - // TODO Auto-generated method stub - } - - @Override - public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) { - if (inputVector.isRepeating) { - inputElementNum = 0; - } - if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) { - vector[outElementNum].set( - ((DecimalColumnVector) inputVector).vector[inputElementNum], - precision, scale); - if (!vector[outElementNum].isSet()) { - isNull[outElementNum] = true; - noNulls = false; - } else { - isNull[outElementNum] = false; - } - } else { - isNull[outElementNum] = true; - noNulls = false; - } - } - - @Override - public void stringifyValue(StringBuilder buffer, int row) { - if (isRepeating) { - row = 0; - } - if (noNulls || !isNull[row]) { - buffer.append(vector[row].toString()); - } else { - buffer.append("null"); - } - } - - public void set(int elementNum, HiveDecimalWritable writeable) { - vector[elementNum].set(writeable, precision, scale); - if (!vector[elementNum].isSet()) { - noNulls = false; - isNull[elementNum] = true; - } else { - isNull[elementNum] = false; - } - } - - public void set(int elementNum, HiveDecimal hiveDec) { - vector[elementNum].set(hiveDec, precision, scale); - if (!vector[elementNum].isSet()) { - noNulls = false; - isNull[elementNum] = true; - } else { - isNull[elementNum] = false; - } - } - - public void setNullDataValue(int elementNum) { - // E.g. For scale 2 the minimum is "0.01" - vector[elementNum].setFromLongAndScale(1L, scale); - } - - @Override - public void ensureSize(int size, boolean preserveData) { - super.ensureSize(size, preserveData); - if (size <= vector.length) return; // We assume the existing vector is always valid. - HiveDecimalWritable[] oldArray = vector; - vector = new HiveDecimalWritable[size]; - int initPos = 0; - if (preserveData) { - // we copy all of the values to avoid creating more objects - // TODO: it might be cheaper to always preserve data or reset existing objects - initPos = oldArray.length; - System.arraycopy(oldArray, 0, vector, 0 , oldArray.length); - } - for (int i = initPos; i < vector.length; ++i) { - vector[i] = new HiveDecimalWritable(0); // Initially zero. - } - } - - @Override - public void shallowCopyTo(ColumnVector otherCv) { - DecimalColumnVector other = (DecimalColumnVector)otherCv; - super.shallowCopyTo(other); - other.scale = scale; - other.precision = precision; - other.vector = vector; - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java deleted file mode 100644 index 1395144ddde598508995d0a1077b8c19d2d6c510..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java +++ /dev/null @@ -1,184 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.exec.vector; - -import java.util.Arrays; - -/** - * This class represents a nullable double precision floating point column vector. - * This class will be used for operations on all floating point types (float, double) - * and as such will use a 64-bit double value to hold the biggest possible value. - * During copy-in/copy-out, smaller types (i.e. float) will be converted as needed. This will - * reduce the amount of code that needs to be generated and also will run fast since the - * machine operates with 64-bit words. - * - * The vector[] field is public by design for high-performance access in the inner - * loop of query execution. - */ -public class DoubleColumnVector extends ColumnVector { - public double[] vector; - public static final double NULL_VALUE = Double.NaN; - - /** - * Use this constructor by default. All column vectors - * should normally be the default size. - */ - public DoubleColumnVector() { - this(VectorizedRowBatch.DEFAULT_SIZE); - } - - /** - * Don't use this except for testing purposes. - * - * @param len - */ - public DoubleColumnVector(int len) { - super(Type.DOUBLE, len); - vector = new double[len]; - } - - // Copy the current object contents into the output. Only copy selected entries, - // as indicated by selectedInUse and the sel array. - public void copySelected( - boolean selectedInUse, int[] sel, int size, DoubleColumnVector output) { - - // Output has nulls if and only if input has nulls. - output.noNulls = noNulls; - output.isRepeating = false; - - // Handle repeating case - if (isRepeating) { - output.vector[0] = vector[0]; - output.isNull[0] = isNull[0]; - output.isRepeating = true; - return; - } - - // Handle normal case - - // Copy data values over - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - output.vector[i] = vector[i]; - } - } - else { - System.arraycopy(vector, 0, output.vector, 0, size); - } - - // Copy nulls over if needed - if (!noNulls) { - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - output.isNull[i] = isNull[i]; - } - } - else { - System.arraycopy(isNull, 0, output.isNull, 0, size); - } - } - } - - // Fill the column vector with the provided value - public void fill(double value) { - noNulls = true; - isRepeating = true; - vector[0] = value; - } - - // Fill the column vector with nulls - public void fillWithNulls() { - noNulls = false; - isRepeating = true; - vector[0] = NULL_VALUE; - isNull[0] = true; - } - - // Simplify vector by brute-force flattening noNulls and isRepeating - // This can be used to reduce combinatorial explosion of code paths in VectorExpressions - // with many arguments. - public void flatten(boolean selectedInUse, int[] sel, int size) { - flattenPush(); - if (isRepeating) { - isRepeating = false; - double repeatVal = vector[0]; - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - vector[i] = repeatVal; - } - } else { - Arrays.fill(vector, 0, size, repeatVal); - } - flattenRepeatingNulls(selectedInUse, sel, size); - } - flattenNoNulls(selectedInUse, sel, size); - } - - @Override - public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) { - if (inputVector.isRepeating) { - inputElementNum = 0; - } - if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) { - isNull[outElementNum] = false; - vector[outElementNum] = - ((DoubleColumnVector) inputVector).vector[inputElementNum]; - } else { - isNull[outElementNum] = true; - noNulls = false; - } - } - - @Override - public void stringifyValue(StringBuilder buffer, int row) { - if (isRepeating) { - row = 0; - } - if (noNulls || !isNull[row]) { - buffer.append(vector[row]); - } else { - buffer.append("null"); - } - } - - @Override - public void ensureSize(int size, boolean preserveData) { - super.ensureSize(size, preserveData); - if (size > vector.length) { - double[] oldArray = vector; - vector = new double[size]; - if (preserveData) { - if (isRepeating) { - vector[0] = oldArray[0]; - } else { - System.arraycopy(oldArray, 0, vector, 0 , oldArray.length); - } - } - } - } - - @Override - public void shallowCopyTo(ColumnVector otherCv) { - DoubleColumnVector other = (DoubleColumnVector)otherCv; - super.shallowCopyTo(other); - other.vector = vector; - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java deleted file mode 100644 index 587e2b9ac8454174109c3b437e539876bc5b6ce0..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java +++ /dev/null @@ -1,375 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.exec.vector; - -import java.util.Arrays; - -import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; -import org.apache.hadoop.io.Writable; - -/** - * This class represents a nullable interval day time column vector capable of handing a - * wide range of interval day time values. - * - * We store the 2 (value) fields of a HiveIntervalDayTime class in primitive arrays. - * - * We do this to avoid an array of Java HiveIntervalDayTime objects which would have poor storage - * and memory access characteristics. - * - * Generally, the caller will fill in a scratch HiveIntervalDayTime object with values from a row, - * work using the scratch HiveIntervalDayTime, and then perhaps update the column vector row - * with a result. - */ -public class IntervalDayTimeColumnVector extends ColumnVector { - - /* - * The storage arrays for this column vector corresponds to the storage of a HiveIntervalDayTime: - */ - private long[] totalSeconds; - // The values from HiveIntervalDayTime.getTotalSeconds(). - - private int[] nanos; - // The values from HiveIntervalDayTime.getNanos(). - - /* - * Scratch objects. - */ - private final HiveIntervalDayTime scratchIntervalDayTime; - - private Writable scratchWritable; - // Supports keeping a HiveIntervalDayTimeWritable object without having to import - // that definition... - - /** - * Use this constructor by default. All column vectors - * should normally be the default size. - */ - public IntervalDayTimeColumnVector() { - this(VectorizedRowBatch.DEFAULT_SIZE); - } - - /** - * Don't use this except for testing purposes. - * - * @param len the number of rows - */ - public IntervalDayTimeColumnVector(int len) { - super(Type.INTERVAL_DAY_TIME, len); - - totalSeconds = new long[len]; - nanos = new int[len]; - - scratchIntervalDayTime = new HiveIntervalDayTime(); - - scratchWritable = null; // Allocated by caller. - } - - /** - * Return the number of rows. - * @return - */ - public int getLength() { - return totalSeconds.length; - } - - /** - * Return a row's HiveIntervalDayTime.getTotalSeconds() value. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param elementNum - * @return - */ - public long getTotalSeconds(int elementNum) { - return totalSeconds[elementNum]; - } - - /** - * Return a row's HiveIntervalDayTime.getNanos() value. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param elementNum - * @return - */ - public long getNanos(int elementNum) { - return nanos[elementNum]; - } - - /** - * Return a row's HiveIntervalDayTime.getDouble() value. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param elementNum - * @return - */ - public double getDouble(int elementNum) { - return asScratchIntervalDayTime(elementNum).getDouble(); - } - - /** - * Set a HiveIntervalDayTime object from a row of the column. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param intervalDayTime - * @param elementNum - */ - public void intervalDayTimeUpdate(HiveIntervalDayTime intervalDayTime, int elementNum) { - intervalDayTime.set(totalSeconds[elementNum], nanos[elementNum]); - } - - - /** - * Return the scratch HiveIntervalDayTime object set from a row. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param elementNum - * @return - */ - public HiveIntervalDayTime asScratchIntervalDayTime(int elementNum) { - scratchIntervalDayTime.set(totalSeconds[elementNum], nanos[elementNum]); - return scratchIntervalDayTime; - } - - /** - * Return the scratch HiveIntervalDayTime (contents undefined). - * @return - */ - public HiveIntervalDayTime getScratchIntervalDayTime() { - return scratchIntervalDayTime; - } - - /** - * Compare row to HiveIntervalDayTime. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param elementNum - * @param intervalDayTime - * @return -1, 0, 1 standard compareTo values. - */ - public int compareTo(int elementNum, HiveIntervalDayTime intervalDayTime) { - return asScratchIntervalDayTime(elementNum).compareTo(intervalDayTime); - } - - /** - * Compare HiveIntervalDayTime to row. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param intervalDayTime - * @param elementNum - * @return -1, 0, 1 standard compareTo values. - */ - public int compareTo(HiveIntervalDayTime intervalDayTime, int elementNum) { - return intervalDayTime.compareTo(asScratchIntervalDayTime(elementNum)); - } - - /** - * Compare a row to another TimestampColumnVector's row. - * @param elementNum1 - * @param intervalDayTimeColVector2 - * @param elementNum2 - * @return - */ - public int compareTo(int elementNum1, IntervalDayTimeColumnVector intervalDayTimeColVector2, - int elementNum2) { - return asScratchIntervalDayTime(elementNum1).compareTo( - intervalDayTimeColVector2.asScratchIntervalDayTime(elementNum2)); - } - - /** - * Compare another TimestampColumnVector's row to a row. - * @param intervalDayTimeColVector1 - * @param elementNum1 - * @param elementNum2 - * @return - */ - public int compareTo(IntervalDayTimeColumnVector intervalDayTimeColVector1, int elementNum1, - int elementNum2) { - return intervalDayTimeColVector1.asScratchIntervalDayTime(elementNum1).compareTo( - asScratchIntervalDayTime(elementNum2)); - } - - @Override - public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) { - - IntervalDayTimeColumnVector timestampColVector = (IntervalDayTimeColumnVector) inputVector; - - totalSeconds[outElementNum] = timestampColVector.totalSeconds[inputElementNum]; - nanos[outElementNum] = timestampColVector.nanos[inputElementNum]; - } - - // Simplify vector by brute-force flattening noNulls and isRepeating - // This can be used to reduce combinatorial explosion of code paths in VectorExpressions - // with many arguments. - public void flatten(boolean selectedInUse, int[] sel, int size) { - flattenPush(); - if (isRepeating) { - isRepeating = false; - long repeatFastTime = totalSeconds[0]; - int repeatNanos = nanos[0]; - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - totalSeconds[i] = repeatFastTime; - nanos[i] = repeatNanos; - } - } else { - Arrays.fill(totalSeconds, 0, size, repeatFastTime); - Arrays.fill(nanos, 0, size, repeatNanos); - } - flattenRepeatingNulls(selectedInUse, sel, size); - } - flattenNoNulls(selectedInUse, sel, size); - } - - /** - * Set a row from a HiveIntervalDayTime. - * We assume the entry has already been isRepeated adjusted. - * @param elementNum - * @param intervalDayTime - */ - public void set(int elementNum, HiveIntervalDayTime intervalDayTime) { - this.totalSeconds[elementNum] = intervalDayTime.getTotalSeconds(); - this.nanos[elementNum] = intervalDayTime.getNanos(); - } - - /** - * Set a row from the current value in the scratch interval day time. - * @param elementNum - */ - public void setFromScratchIntervalDayTime(int elementNum) { - this.totalSeconds[elementNum] = scratchIntervalDayTime.getTotalSeconds(); - this.nanos[elementNum] = scratchIntervalDayTime.getNanos(); - } - - /** - * Set row to standard null value(s). - * We assume the entry has already been isRepeated adjusted. - * @param elementNum - */ - public void setNullValue(int elementNum) { - totalSeconds[elementNum] = 0; - nanos[elementNum] = 1; - } - - // Copy the current object contents into the output. Only copy selected entries, - // as indicated by selectedInUse and the sel array. - public void copySelected( - boolean selectedInUse, int[] sel, int size, IntervalDayTimeColumnVector output) { - - // Output has nulls if and only if input has nulls. - output.noNulls = noNulls; - output.isRepeating = false; - - // Handle repeating case - if (isRepeating) { - output.totalSeconds[0] = totalSeconds[0]; - output.nanos[0] = nanos[0]; - output.isNull[0] = isNull[0]; - output.isRepeating = true; - return; - } - - // Handle normal case - - // Copy data values over - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - output.totalSeconds[i] = totalSeconds[i]; - output.nanos[i] = nanos[i]; - } - } - else { - System.arraycopy(totalSeconds, 0, output.totalSeconds, 0, size); - System.arraycopy(nanos, 0, output.nanos, 0, size); - } - - // Copy nulls over if needed - if (!noNulls) { - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - output.isNull[i] = isNull[i]; - } - } - else { - System.arraycopy(isNull, 0, output.isNull, 0, size); - } - } - } - - /** - * Fill all the vector entries with a HiveIntervalDayTime. - * @param intervalDayTime - */ - public void fill(HiveIntervalDayTime intervalDayTime) { - noNulls = true; - isRepeating = true; - totalSeconds[0] = intervalDayTime.getTotalSeconds(); - nanos[0] = intervalDayTime.getNanos(); - } - - /** - * Return a convenience writable object stored by this column vector. - * Supports keeping a TimestampWritable object without having to import that definition... - * @return - */ - public Writable getScratchWritable() { - return scratchWritable; - } - - /** - * Set the convenience writable object stored by this column vector - * @param scratchWritable - */ - public void setScratchWritable(Writable scratchWritable) { - this.scratchWritable = scratchWritable; - } - - @Override - public void stringifyValue(StringBuilder buffer, int row) { - if (isRepeating) { - row = 0; - } - if (noNulls || !isNull[row]) { - scratchIntervalDayTime.set(totalSeconds[row], nanos[row]); - buffer.append(scratchIntervalDayTime.toString()); - } else { - buffer.append("null"); - } - } - - @Override - public void ensureSize(int size, boolean preserveData) { - super.ensureSize(size, preserveData); - if (size <= totalSeconds.length) return; - long[] oldTime = totalSeconds; - int[] oldNanos = nanos; - totalSeconds = new long[size]; - nanos = new int[size]; - if (preserveData) { - if (isRepeating) { - totalSeconds[0] = oldTime[0]; - nanos[0] = oldNanos[0]; - } else { - System.arraycopy(oldTime, 0, totalSeconds, 0, oldTime.length); - System.arraycopy(oldNanos, 0, nanos, 0, oldNanos.length); - } - } - } - - @Override - public void shallowCopyTo(ColumnVector otherCv) { - IntervalDayTimeColumnVector other = (IntervalDayTimeColumnVector)otherCv; - super.shallowCopyTo(other); - other.totalSeconds = totalSeconds; - other.nanos = nanos; - } -} \ No newline at end of file diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ListColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ListColumnVector.java deleted file mode 100644 index 02a8b3cec5f5cad98e4802e6e1f1e9ece268921f..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ListColumnVector.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector; - -/** - * The representation of a vectorized column of list objects. - * - * Each list is composed of a range of elements in the underlying child - * ColumnVector. The range for list i is - * offsets[i]..offsets[i]+lengths[i]-1 inclusive. - */ -public class ListColumnVector extends MultiValuedColumnVector { - - public ColumnVector child; - - public ListColumnVector() { - this(VectorizedRowBatch.DEFAULT_SIZE, null); - } - - /** - * Constructor for ListColumnVector. - * - * @param len Vector length - * @param child The child vector - */ - public ListColumnVector(int len, ColumnVector child) { - super(Type.LIST, len); - this.child = child; - } - - @Override - protected void childFlatten(boolean useSelected, int[] selected, int size) { - child.flatten(useSelected, selected, size); - } - - @Override - public void setElement(int outElementNum, int inputElementNum, - ColumnVector inputVector) { - ListColumnVector input = (ListColumnVector) inputVector; - if (input.isRepeating) { - inputElementNum = 0; - } - if (!input.noNulls && input.isNull[inputElementNum]) { - isNull[outElementNum] = true; - noNulls = false; - } else { - isNull[outElementNum] = false; - int offset = childCount; - int length = (int) input.lengths[inputElementNum]; - int inputOffset = (int) input.offsets[inputElementNum]; - offsets[outElementNum] = offset; - childCount += length; - lengths[outElementNum] = length; - child.ensureSize(childCount, true); - for (int i = 0; i < length; ++i) { - child.setElement(i + offset, inputOffset + i, input.child); - } - } - } - - @Override - public void stringifyValue(StringBuilder buffer, int row) { - if (isRepeating) { - row = 0; - } - if (noNulls || !isNull[row]) { - buffer.append('['); - boolean isFirst = true; - for(long i=offsets[row]; i < offsets[row] + lengths[row]; ++i) { - if (isFirst) { - isFirst = false; - } else { - buffer.append(", "); - } - child.stringifyValue(buffer, (int) i); - } - buffer.append(']'); - } else { - buffer.append("null"); - } - } - - @Override - public void init() { - super.init(); - child.init(); - } - - @Override - public void reset() { - super.reset(); - child.reset(); - } - - @Override - public void unFlatten() { - super.unFlatten(); - if (!isRepeating || noNulls || !isNull[0]) { - child.unFlatten(); - } - } - -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java deleted file mode 100644 index a407bcdaaaa84e957725783cf84fec06ddcf7143..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java +++ /dev/null @@ -1,230 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.exec.vector; - -import java.util.Arrays; - -/** - * This class represents a nullable int column vector. - * This class will be used for operations on all integer types (tinyint, smallint, int, bigint) - * and as such will use a 64-bit long value to hold the biggest possible value. - * During copy-in/copy-out, smaller int types will be converted as needed. This will - * reduce the amount of code that needs to be generated and also will run fast since the - * machine operates with 64-bit words. - * - * The vector[] field is public by design for high-performance access in the inner - * loop of query execution. - */ -public class LongColumnVector extends ColumnVector { - public long[] vector; - public static final long NULL_VALUE = 1; - - /** - * Use this constructor by default. All column vectors - * should normally be the default size. - */ - public LongColumnVector() { - this(VectorizedRowBatch.DEFAULT_SIZE); - } - - /** - * Don't use this except for testing purposes. - * - * @param len the number of rows - */ - public LongColumnVector(int len) { - super(Type.LONG, len); - vector = new long[len]; - } - - // Copy the current object contents into the output. Only copy selected entries, - // as indicated by selectedInUse and the sel array. - public void copySelected( - boolean selectedInUse, int[] sel, int size, LongColumnVector output) { - - // Output has nulls if and only if input has nulls. - output.noNulls = noNulls; - output.isRepeating = false; - - // Handle repeating case - if (isRepeating) { - output.vector[0] = vector[0]; - output.isNull[0] = isNull[0]; - output.isRepeating = true; - return; - } - - // Handle normal case - - // Copy data values over - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - output.vector[i] = vector[i]; - } - } - else { - System.arraycopy(vector, 0, output.vector, 0, size); - } - - // Copy nulls over if needed - if (!noNulls) { - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - output.isNull[i] = isNull[i]; - } - } - else { - System.arraycopy(isNull, 0, output.isNull, 0, size); - } - } - } - - // Copy the current object contents into the output. Only copy selected entries, - // as indicated by selectedInUse and the sel array. - public void copySelected( - boolean selectedInUse, int[] sel, int size, DoubleColumnVector output) { - - // Output has nulls if and only if input has nulls. - output.noNulls = noNulls; - output.isRepeating = false; - - // Handle repeating case - if (isRepeating) { - output.vector[0] = vector[0]; // automatic conversion to double is done here - output.isNull[0] = isNull[0]; - output.isRepeating = true; - return; - } - - // Handle normal case - - // Copy data values over - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - output.vector[i] = vector[i]; - } - } - else { - for(int i = 0; i < size; ++i) { - output.vector[i] = vector[i]; - } - } - - // Copy nulls over if needed - if (!noNulls) { - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - output.isNull[i] = isNull[i]; - } - } - else { - System.arraycopy(isNull, 0, output.isNull, 0, size); - } - } - } - - // Fill the column vector with the provided value - public void fill(long value) { - noNulls = true; - isRepeating = true; - vector[0] = value; - } - - // Fill the column vector with nulls - public void fillWithNulls() { - noNulls = false; - isRepeating = true; - vector[0] = NULL_VALUE; - isNull[0] = true; - } - - // Simplify vector by brute-force flattening noNulls and isRepeating - // This can be used to reduce combinatorial explosion of code paths in VectorExpressions - // with many arguments. - public void flatten(boolean selectedInUse, int[] sel, int size) { - flattenPush(); - if (isRepeating) { - isRepeating = false; - long repeatVal = vector[0]; - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - vector[i] = repeatVal; - } - } else { - Arrays.fill(vector, 0, size, repeatVal); - } - flattenRepeatingNulls(selectedInUse, sel, size); - } - flattenNoNulls(selectedInUse, sel, size); - } - - @Override - public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) { - if (inputVector.isRepeating) { - inputElementNum = 0; - } - if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) { - isNull[outElementNum] = false; - vector[outElementNum] = - ((LongColumnVector) inputVector).vector[inputElementNum]; - } else { - isNull[outElementNum] = true; - noNulls = false; - } - } - - @Override - public void stringifyValue(StringBuilder buffer, int row) { - if (isRepeating) { - row = 0; - } - if (noNulls || !isNull[row]) { - buffer.append(vector[row]); - } else { - buffer.append("null"); - } - } - - @Override - public void ensureSize(int size, boolean preserveData) { - super.ensureSize(size, preserveData); - if (size > vector.length) { - long[] oldArray = vector; - vector = new long[size]; - if (preserveData) { - if (isRepeating) { - vector[0] = oldArray[0]; - } else { - System.arraycopy(oldArray, 0, vector, 0 , oldArray.length); - } - } - } - } - - @Override - public void shallowCopyTo(ColumnVector otherCv) { - LongColumnVector other = (LongColumnVector)otherCv; - super.shallowCopyTo(other); - other.vector = vector; - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MapColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MapColumnVector.java deleted file mode 100644 index 64badb99e5385af359691bc1d4df5119c9ffaac3..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MapColumnVector.java +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector; - -/** - * The representation of a vectorized column of map objects. - * - * Each map is composed of a range of elements in the underlying child - * ColumnVector. The range for map i is - * offsets[i]..offsets[i]+lengths[i]-1 inclusive. - */ -public class MapColumnVector extends MultiValuedColumnVector { - - public ColumnVector keys; - public ColumnVector values; - - public MapColumnVector() { - this(VectorizedRowBatch.DEFAULT_SIZE, null, null); - } - - /** - * Constructor for MapColumnVector - * - * @param len Vector length - * @param keys The keys column vector - * @param values The values column vector - */ - public MapColumnVector(int len, ColumnVector keys, ColumnVector values) { - super(Type.MAP, len); - this.keys = keys; - this.values = values; - } - - @Override - protected void childFlatten(boolean useSelected, int[] selected, int size) { - keys.flatten(useSelected, selected, size); - values.flatten(useSelected, selected, size); - } - - @Override - public void setElement(int outElementNum, int inputElementNum, - ColumnVector inputVector) { - if (inputVector.isRepeating) { - inputElementNum = 0; - } - if (!inputVector.noNulls && inputVector.isNull[inputElementNum]) { - isNull[outElementNum] = true; - noNulls = false; - } else { - MapColumnVector input = (MapColumnVector) inputVector; - isNull[outElementNum] = false; - int offset = childCount; - int length = (int) input.lengths[inputElementNum]; - int inputOffset = (int) input.offsets[inputElementNum]; - offsets[outElementNum] = offset; - childCount += length; - lengths[outElementNum] = length; - keys.ensureSize(childCount, true); - values.ensureSize(childCount, true); - for (int i = 0; i < length; ++i) { - keys.setElement(i + offset, inputOffset + i, input.keys); - values.setElement(i + offset, inputOffset + i, input.values); - } - } - } - - @Override - public void stringifyValue(StringBuilder buffer, int row) { - if (isRepeating) { - row = 0; - } - if (noNulls || !isNull[row]) { - buffer.append('['); - boolean isFirst = true; - for(long i=offsets[row]; i < offsets[row] + lengths[row]; ++i) { - if (isFirst) { - isFirst = false; - } else { - buffer.append(", "); - } - buffer.append("{\"key\": "); - keys.stringifyValue(buffer, (int) i); - buffer.append(", \"value\": "); - values.stringifyValue(buffer, (int) i); - buffer.append('}'); - } - buffer.append(']'); - } else { - buffer.append("null"); - } - } - - @Override - public void init() { - super.init(); - keys.init(); - values.init(); - } - - @Override - public void reset() { - super.reset(); - keys.reset(); - values.reset(); - } - - @Override - public void unFlatten() { - super.unFlatten(); - if (!isRepeating || noNulls || !isNull[0]) { - keys.unFlatten(); - values.unFlatten(); - } - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MultiValuedColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MultiValuedColumnVector.java deleted file mode 100644 index 66136351d4f508d1dc7cf419da8eca41efa698a1..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MultiValuedColumnVector.java +++ /dev/null @@ -1,154 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector; - -import java.util.Arrays; - -/** - * The representation of a vectorized column of multi-valued objects, such - * as lists and maps. - * - * Each object is composed of a range of elements in the underlying child - * ColumnVector. The range for list i is - * offsets[i]..offsets[i]+lengths[i]-1 inclusive. - */ -public abstract class MultiValuedColumnVector extends ColumnVector { - - public long[] offsets; - public long[] lengths; - // the number of children slots used - public int childCount; - - /** - * Constructor for MultiValuedColumnVector. - * - * @param len Vector length - */ - public MultiValuedColumnVector(Type type, int len) { - super(type, len); - childCount = 0; - offsets = new long[len]; - lengths = new long[len]; - } - - protected abstract void childFlatten(boolean useSelected, int[] selected, - int size); - - @Override - public void flatten(boolean selectedInUse, int[] sel, int size) { - flattenPush(); - - if (isRepeating) { - if (noNulls || !isNull[0]) { - if (selectedInUse) { - for (int i = 0; i < size; ++i) { - int row = sel[i]; - offsets[row] = offsets[0]; - lengths[row] = lengths[0]; - isNull[row] = false; - } - } else { - Arrays.fill(offsets, 0, size, offsets[0]); - Arrays.fill(lengths, 0, size, lengths[0]); - Arrays.fill(isNull, 0, size, false); - } - // We optimize by assuming that a repeating list/map will run from - // from 0 .. lengths[0] in the child vector. - // Sanity check the assumption that we can start at 0. - if (offsets[0] != 0) { - throw new IllegalArgumentException("Repeating offset isn't 0, but " + - offsets[0]); - } - childFlatten(false, null, (int) lengths[0]); - } else { - if (selectedInUse) { - for(int i=0; i < size; ++i) { - isNull[sel[i]] = true; - } - } else { - Arrays.fill(isNull, 0, size, true); - } - } - isRepeating = false; - noNulls = false; - } else { - if (selectedInUse) { - int childSize = 0; - for(int i=0; i < size; ++i) { - childSize += lengths[sel[i]]; - } - int[] childSelection = new int[childSize]; - int idx = 0; - for(int i=0; i < size; ++i) { - int row = sel[i]; - for(int elem=0; elem < lengths[row]; ++elem) { - childSelection[idx++] = (int) (offsets[row] + elem); - } - } - childFlatten(true, childSelection, childSize); - } else { - childFlatten(false, null, childCount); - } - flattenNoNulls(selectedInUse, sel, size); - } - } - - @Override - public void ensureSize(int size, boolean preserveData) { - super.ensureSize(size, preserveData); - if (size > offsets.length) { - long[] oldOffsets = offsets; - offsets = new long[size]; - long oldLengths[] = lengths; - lengths = new long[size]; - if (preserveData) { - if (isRepeating) { - offsets[0] = oldOffsets[0]; - lengths[0] = oldLengths[0]; - } else { - System.arraycopy(oldOffsets, 0, offsets, 0 , oldOffsets.length); - System.arraycopy(oldLengths, 0, lengths, 0, oldLengths.length); - } - } - } - } - - /** - * Initializee the vector - */ - @Override - public void init() { - super.init(); - childCount = 0; - } - - /** - * Reset the vector for the next batch. - */ - @Override - public void reset() { - super.reset(); - childCount = 0; - } - - @Override - public void shallowCopyTo(ColumnVector otherCv) { - throw new UnsupportedOperationException(); // Implement in future, if needed. - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java deleted file mode 100644 index 45f3ac6d6eb0969c96d461f0cf532f0a783ab61d..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector; - -/** - * The representation of a vectorized column of struct objects. - * - * Each field is represented by a separate inner ColumnVector. Since this - * ColumnVector doesn't own any per row data other that the isNull flag, the - * isRepeating only covers the isNull array. - */ -public class StructColumnVector extends ColumnVector { - - public ColumnVector[] fields; - - public StructColumnVector() { - this(VectorizedRowBatch.DEFAULT_SIZE); - } - - /** - * Constructor for StructColumnVector - * - * @param len Vector length - * @param fields the field column vectors - */ - public StructColumnVector(int len, ColumnVector... fields) { - super(Type.STRUCT, len); - this.fields = fields; - } - - @Override - public void flatten(boolean selectedInUse, int[] sel, int size) { - flattenPush(); - for(int i=0; i < fields.length; ++i) { - fields[i].flatten(selectedInUse, sel, size); - } - flattenNoNulls(selectedInUse, sel, size); - } - - @Override - public void setElement(int outElementNum, int inputElementNum, - ColumnVector inputVector) { - if (inputVector.isRepeating) { - inputElementNum = 0; - } - if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) { - isNull[outElementNum] = false; - ColumnVector[] inputFields = ((StructColumnVector) inputVector).fields; - for (int i = 0; i < inputFields.length; ++i) { - fields[i].setElement(outElementNum, inputElementNum, inputFields[i]); - } - } else { - noNulls = false; - isNull[outElementNum] = true; - } - } - - @Override - public void stringifyValue(StringBuilder buffer, int row) { - if (isRepeating) { - row = 0; - } - if (noNulls || !isNull[row]) { - buffer.append('['); - for(int i=0; i < fields.length; ++i) { - if (i != 0) { - buffer.append(", "); - } - fields[i].stringifyValue(buffer, row); - } - buffer.append(']'); - } else { - buffer.append("null"); - } - } - - @Override - public void ensureSize(int size, boolean preserveData) { - super.ensureSize(size, preserveData); - for(int i=0; i < fields.length; ++i) { - fields[i].ensureSize(size, preserveData); - } - } - - @Override - public void reset() { - super.reset(); - for(int i =0; i < fields.length; ++i) { - fields[i].reset(); - } - } - - @Override - public void init() { - super.init(); - for(int i =0; i < fields.length; ++i) { - fields[i].init(); - } - } - - @Override - public void unFlatten() { - super.unFlatten(); - for(int i=0; i < fields.length; ++i) { - fields[i].unFlatten(); - } - } - - @Override - public void setRepeating(boolean isRepeating) { - super.setRepeating(isRepeating); - for(int i=0; i < fields.length; ++i) { - fields[i].setRepeating(isRepeating); - } - } - - @Override - public void shallowCopyTo(ColumnVector otherCv) { - throw new UnsupportedOperationException(); // Implement if needed. - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java deleted file mode 100644 index ef1c817efaac126aab0a35b013aa273c1a70f49f..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java +++ /dev/null @@ -1,427 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.exec.vector; - -import java.sql.Timestamp; -import java.util.Arrays; - -import org.apache.hadoop.io.Writable; - -/** - * This class represents a nullable timestamp column vector capable of handing a wide range of - * timestamp values. - * - * We store the 2 (value) fields of a Timestamp class in primitive arrays. - * - * We do this to avoid an array of Java Timestamp objects which would have poor storage - * and memory access characteristics. - * - * Generally, the caller will fill in a scratch timestamp object with values from a row, work - * using the scratch timestamp, and then perhaps update the column vector row with a result. - */ -public class TimestampColumnVector extends ColumnVector { - - /* - * The storage arrays for this column vector corresponds to the storage of a Timestamp: - */ - public long[] time; - // The values from Timestamp.getTime(). - - public int[] nanos; - // The values from Timestamp.getNanos(). - - /* - * Scratch objects. - */ - private final Timestamp scratchTimestamp; - - private Writable scratchWritable; - // Supports keeping a TimestampWritable object without having to import that definition... - - /** - * Use this constructor by default. All column vectors - * should normally be the default size. - */ - public TimestampColumnVector() { - this(VectorizedRowBatch.DEFAULT_SIZE); - } - - /** - * Don't use this except for testing purposes. - * - * @param len the number of rows - */ - public TimestampColumnVector(int len) { - super(Type.TIMESTAMP, len); - - time = new long[len]; - nanos = new int[len]; - - scratchTimestamp = new Timestamp(0); - - scratchWritable = null; // Allocated by caller. - } - - /** - * Return the number of rows. - * @return - */ - public int getLength() { - return time.length; - } - - /** - * Return a row's Timestamp.getTime() value. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param elementNum - * @return - */ - public long getTime(int elementNum) { - return time[elementNum]; - } - - /** - * Return a row's Timestamp.getNanos() value. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param elementNum - * @return - */ - public int getNanos(int elementNum) { - return nanos[elementNum]; - } - - /** - * Set a Timestamp object from a row of the column. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param timestamp - * @param elementNum - */ - public void timestampUpdate(Timestamp timestamp, int elementNum) { - timestamp.setTime(time[elementNum]); - timestamp.setNanos(nanos[elementNum]); - } - - /** - * Return the scratch Timestamp object set from a row. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param elementNum - * @return - */ - public Timestamp asScratchTimestamp(int elementNum) { - scratchTimestamp.setTime(time[elementNum]); - scratchTimestamp.setNanos(nanos[elementNum]); - return scratchTimestamp; - } - - /** - * Return the scratch timestamp (contents undefined). - * @return - */ - public Timestamp getScratchTimestamp() { - return scratchTimestamp; - } - - /** - * Return a long representation of a Timestamp. - * @param elementNum - * @return - */ - public long getTimestampAsLong(int elementNum) { - scratchTimestamp.setTime(time[elementNum]); - scratchTimestamp.setNanos(nanos[elementNum]); - return getTimestampAsLong(scratchTimestamp); - } - - /** - * Return a long representation of a Timestamp. - * @param timestamp - * @return - */ - public static long getTimestampAsLong(Timestamp timestamp) { - return millisToSeconds(timestamp.getTime()); - } - - // Copy of TimestampWritable.millisToSeconds - /** - * Rounds the number of milliseconds relative to the epoch down to the nearest whole number of - * seconds. 500 would round to 0, -500 would round to -1. - */ - private static long millisToSeconds(long millis) { - if (millis >= 0) { - return millis / 1000; - } else { - return (millis - 999) / 1000; - } - } - - /** - * Return a double representation of a Timestamp. - * @param elementNum - * @return - */ - public double getDouble(int elementNum) { - scratchTimestamp.setTime(time[elementNum]); - scratchTimestamp.setNanos(nanos[elementNum]); - return getDouble(scratchTimestamp); - } - - /** - * Return a double representation of a Timestamp. - * @param timestamp - * @return - */ - public static double getDouble(Timestamp timestamp) { - // Same algorithm as TimestampWritable (not currently import-able here). - double seconds, nanos; - seconds = millisToSeconds(timestamp.getTime()); - nanos = timestamp.getNanos(); - return seconds + nanos / 1000000000; - } - - /** - * Compare row to Timestamp. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param elementNum - * @param timestamp - * @return -1, 0, 1 standard compareTo values. - */ - public int compareTo(int elementNum, Timestamp timestamp) { - return asScratchTimestamp(elementNum).compareTo(timestamp); - } - - /** - * Compare Timestamp to row. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param timestamp - * @param elementNum - * @return -1, 0, 1 standard compareTo values. - */ - public int compareTo(Timestamp timestamp, int elementNum) { - return timestamp.compareTo(asScratchTimestamp(elementNum)); - } - - /** - * Compare a row to another TimestampColumnVector's row. - * @param elementNum1 - * @param timestampColVector2 - * @param elementNum2 - * @return - */ - public int compareTo(int elementNum1, TimestampColumnVector timestampColVector2, - int elementNum2) { - return asScratchTimestamp(elementNum1).compareTo( - timestampColVector2.asScratchTimestamp(elementNum2)); - } - - /** - * Compare another TimestampColumnVector's row to a row. - * @param timestampColVector1 - * @param elementNum1 - * @param elementNum2 - * @return - */ - public int compareTo(TimestampColumnVector timestampColVector1, int elementNum1, - int elementNum2) { - return timestampColVector1.asScratchTimestamp(elementNum1).compareTo( - asScratchTimestamp(elementNum2)); - } - - @Override - public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) { - - TimestampColumnVector timestampColVector = (TimestampColumnVector) inputVector; - - time[outElementNum] = timestampColVector.time[inputElementNum]; - nanos[outElementNum] = timestampColVector.nanos[inputElementNum]; - } - - // Simplify vector by brute-force flattening noNulls and isRepeating - // This can be used to reduce combinatorial explosion of code paths in VectorExpressions - // with many arguments. - public void flatten(boolean selectedInUse, int[] sel, int size) { - flattenPush(); - if (isRepeating) { - isRepeating = false; - long repeatFastTime = time[0]; - int repeatNanos = nanos[0]; - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - time[i] = repeatFastTime; - nanos[i] = repeatNanos; - } - } else { - Arrays.fill(time, 0, size, repeatFastTime); - Arrays.fill(nanos, 0, size, repeatNanos); - } - flattenRepeatingNulls(selectedInUse, sel, size); - } - flattenNoNulls(selectedInUse, sel, size); - } - - /** - * Set a row from a timestamp. - * We assume the entry has already been isRepeated adjusted. - * @param elementNum - * @param timestamp - */ - public void set(int elementNum, Timestamp timestamp) { - if (timestamp == null) { - this.noNulls = false; - this.isNull[elementNum] = true; - } else { - this.time[elementNum] = timestamp.getTime(); - this.nanos[elementNum] = timestamp.getNanos(); - } - } - - /** - * Set a row from the current value in the scratch timestamp. - * @param elementNum - */ - public void setFromScratchTimestamp(int elementNum) { - this.time[elementNum] = scratchTimestamp.getTime(); - this.nanos[elementNum] = scratchTimestamp.getNanos(); - } - - /** - * Set row to standard null value(s). - * We assume the entry has already been isRepeated adjusted. - * @param elementNum - */ - public void setNullValue(int elementNum) { - time[elementNum] = 0; - nanos[elementNum] = 1; - } - - // Copy the current object contents into the output. Only copy selected entries, - // as indicated by selectedInUse and the sel array. - public void copySelected( - boolean selectedInUse, int[] sel, int size, TimestampColumnVector output) { - - // Output has nulls if and only if input has nulls. - output.noNulls = noNulls; - output.isRepeating = false; - - // Handle repeating case - if (isRepeating) { - output.time[0] = time[0]; - output.nanos[0] = nanos[0]; - output.isNull[0] = isNull[0]; - output.isRepeating = true; - return; - } - - // Handle normal case - - // Copy data values over - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - output.time[i] = time[i]; - output.nanos[i] = nanos[i]; - } - } - else { - System.arraycopy(time, 0, output.time, 0, size); - System.arraycopy(nanos, 0, output.nanos, 0, size); - } - - // Copy nulls over if needed - if (!noNulls) { - if (selectedInUse) { - for (int j = 0; j < size; j++) { - int i = sel[j]; - output.isNull[i] = isNull[i]; - } - } - else { - System.arraycopy(isNull, 0, output.isNull, 0, size); - } - } - } - - /** - * Fill all the vector entries with a timestamp. - * @param timestamp - */ - public void fill(Timestamp timestamp) { - noNulls = true; - isRepeating = true; - time[0] = timestamp.getTime(); - nanos[0] = timestamp.getNanos(); - } - - /** - * Return a convenience writable object stored by this column vector. - * Supports keeping a TimestampWritable object without having to import that definition... - * @return - */ - public Writable getScratchWritable() { - return scratchWritable; - } - - /** - * Set the convenience writable object stored by this column vector - * @param scratchWritable - */ - public void setScratchWritable(Writable scratchWritable) { - this.scratchWritable = scratchWritable; - } - - @Override - public void stringifyValue(StringBuilder buffer, int row) { - if (isRepeating) { - row = 0; - } - if (noNulls || !isNull[row]) { - scratchTimestamp.setTime(time[row]); - scratchTimestamp.setNanos(nanos[row]); - buffer.append(scratchTimestamp.toString()); - } else { - buffer.append("null"); - } - } - - @Override - public void ensureSize(int size, boolean preserveData) { - super.ensureSize(size, preserveData); - if (size <= time.length) return; - long[] oldTime = time; - int[] oldNanos = nanos; - time = new long[size]; - nanos = new int[size]; - if (preserveData) { - if (isRepeating) { - time[0] = oldTime[0]; - nanos[0] = oldNanos[0]; - } else { - System.arraycopy(oldTime, 0, time, 0, oldTime.length); - System.arraycopy(oldNanos, 0, nanos, 0, oldNanos.length); - } - } - } - - @Override - public void shallowCopyTo(ColumnVector otherCv) { - TimestampColumnVector other = (TimestampColumnVector)otherCv; - super.shallowCopyTo(other); - other.time = time; - other.nanos = nanos; - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java deleted file mode 100644 index e844b11f9634255f121ab4225e227ef7801adb2e..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java +++ /dev/null @@ -1,145 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector; - -/** - * The representation of a vectorized column of struct objects. - * - * Each field is represented by a separate inner ColumnVector. Since this - * ColumnVector doesn't own any per row data other that the isNull flag, the - * isRepeating only covers the isNull array. - */ -public class UnionColumnVector extends ColumnVector { - - public int[] tags; - public ColumnVector[] fields; - - public UnionColumnVector() { - this(VectorizedRowBatch.DEFAULT_SIZE); - } - - /** - * Constructor for UnionColumnVector - * - * @param len Vector length - * @param fields the field column vectors - */ - public UnionColumnVector(int len, ColumnVector... fields) { - super(Type.UNION, len); - tags = new int[len]; - this.fields = fields; - } - - @Override - public void flatten(boolean selectedInUse, int[] sel, int size) { - flattenPush(); - for(int i=0; i < fields.length; ++i) { - fields[i].flatten(selectedInUse, sel, size); - } - flattenNoNulls(selectedInUse, sel, size); - } - - @Override - public void setElement(int outElementNum, int inputElementNum, - ColumnVector inputVector) { - if (inputVector.isRepeating) { - inputElementNum = 0; - } - if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) { - isNull[outElementNum] = false; - UnionColumnVector input = (UnionColumnVector) inputVector; - tags[outElementNum] = input.tags[inputElementNum]; - fields[tags[outElementNum]].setElement(outElementNum, inputElementNum, - input.fields[tags[outElementNum]]); - } else { - noNulls = false; - isNull[outElementNum] = true; - } - } - - @Override - public void stringifyValue(StringBuilder buffer, int row) { - if (isRepeating) { - row = 0; - } - if (noNulls || !isNull[row]) { - buffer.append("{\"tag\": "); - buffer.append(tags[row]); - buffer.append(", \"value\": "); - fields[tags[row]].stringifyValue(buffer, row); - buffer.append('}'); - } else { - buffer.append("null"); - } - } - - @Override - public void ensureSize(int size, boolean preserveData) { - super.ensureSize(size, preserveData); - if (tags.length < size) { - if (preserveData) { - int[] oldTags = tags; - tags = new int[size]; - System.arraycopy(oldTags, 0, tags, 0, oldTags.length); - } else { - tags = new int[size]; - } - for(int i=0; i < fields.length; ++i) { - fields[i].ensureSize(size, preserveData); - } - } - } - - @Override - public void reset() { - super.reset(); - for(int i =0; i < fields.length; ++i) { - fields[i].reset(); - } - } - - @Override - public void init() { - super.init(); - for(int i =0; i < fields.length; ++i) { - fields[i].init(); - } - } - - @Override - public void unFlatten() { - super.unFlatten(); - for(int i=0; i < fields.length; ++i) { - fields[i].unFlatten(); - } - } - - @Override - public void setRepeating(boolean isRepeating) { - super.setRepeating(isRepeating); - for(int i=0; i < fields.length; ++i) { - fields[i].setRepeating(isRepeating); - } - } - - @Override - public void shallowCopyTo(ColumnVector otherCv) { - throw new UnsupportedOperationException(); // Implement if needed. - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java deleted file mode 100644 index 5e5f13ddc7064aa9343376320e50f2da2c3b20ac..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java +++ /dev/null @@ -1,264 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.exec.vector; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Writable; - -/** - * A VectorizedRowBatch is a set of rows, organized with each column - * as a vector. It is the unit of query execution, organized to minimize - * the cost per row and achieve high cycles-per-instruction. - * The major fields are public by design to allow fast and convenient - * access by the vectorized query execution code. - */ -public class VectorizedRowBatch implements Writable { - public int numCols; // number of columns - public ColumnVector[] cols; // a vector for each column - public int size; // number of rows that qualify (i.e. haven't been filtered out) - public int[] selected; // array of positions of selected values - public int[] projectedColumns; - public int projectionSize; - - private int dataColumnCount; - private int partitionColumnCount; - - - /* - * If no filtering has been applied yet, selectedInUse is false, - * meaning that all rows qualify. If it is true, then the selected[] array - * records the offsets of qualifying rows. - */ - public boolean selectedInUse; - - // If this is true, then there is no data in the batch -- we have hit the end of input. - public boolean endOfFile; - - /* - * This number is carefully chosen to minimize overhead and typically allows - * one VectorizedRowBatch to fit in cache. - */ - public static final int DEFAULT_SIZE = 1024; - - /* - * This number is a safety limit for 32MB of writables. - */ - public static final int DEFAULT_BYTES = 32 * 1024 * 1024; - - /** - * Return a batch with the specified number of columns. - * This is the standard constructor -- all batches should be the same size - * - * @param numCols the number of columns to include in the batch - */ - public VectorizedRowBatch(int numCols) { - this(numCols, DEFAULT_SIZE); - } - - /** - * Return a batch with the specified number of columns and rows. - * Only call this constructor directly for testing purposes. - * Batch size should normally always be defaultSize. - * - * @param numCols the number of columns to include in the batch - * @param size the number of rows to include in the batch - */ - public VectorizedRowBatch(int numCols, int size) { - this.numCols = numCols; - this.size = size; - selected = new int[size]; - selectedInUse = false; - this.cols = new ColumnVector[numCols]; - projectedColumns = new int[numCols]; - - // Initially all columns are projected and in the same order - projectionSize = numCols; - for (int i = 0; i < numCols; i++) { - projectedColumns[i] = i; - } - - dataColumnCount = -1; - partitionColumnCount = -1; - } - - public void setPartitionInfo(int dataColumnCount, int partitionColumnCount) { - this.dataColumnCount = dataColumnCount; - this.partitionColumnCount = partitionColumnCount; - } - - public int getDataColumnCount() { - return dataColumnCount; - } - - public int getPartitionColumnCount() { - return partitionColumnCount; - } - - /** - * Returns the maximum size of the batch (number of rows it can hold) - */ - public int getMaxSize() { - return selected.length; - } - - /** - * Return count of qualifying rows. - * - * @return number of rows that have not been filtered out - */ - public long count() { - return size; - } - - private static String toUTF8(Object o) { - if(o == null || o instanceof NullWritable) { - return "\\N"; /* as found in LazySimpleSerDe's nullSequence */ - } - return o.toString(); - } - - @Override - public String toString() { - if (size == 0) { - return ""; - } - StringBuilder b = new StringBuilder(); - b.append("Column vector types: "); - for (int k = 0; k < projectionSize; k++) { - int projIndex = projectedColumns[k]; - ColumnVector cv = cols[projIndex]; - if (k > 0) { - b.append(", "); - } - b.append(projIndex); - b.append(":"); - String colVectorType = null; - if (cv instanceof LongColumnVector) { - colVectorType = "LONG"; - } else if (cv instanceof DoubleColumnVector) { - colVectorType = "DOUBLE"; - } else if (cv instanceof BytesColumnVector) { - colVectorType = "BYTES"; - } else if (cv instanceof DecimalColumnVector) { - colVectorType = "DECIMAL"; - } else if (cv instanceof TimestampColumnVector) { - colVectorType = "TIMESTAMP"; - } else if (cv instanceof IntervalDayTimeColumnVector) { - colVectorType = "INTERVAL_DAY_TIME"; - } else if (cv instanceof ListColumnVector) { - colVectorType = "LIST"; - } else if (cv instanceof MapColumnVector) { - colVectorType = "MAP"; - } else if (cv instanceof StructColumnVector) { - colVectorType = "STRUCT"; - } else if (cv instanceof UnionColumnVector) { - colVectorType = "UNION"; - } else { - colVectorType = "Unknown"; - } - b.append(colVectorType); - } - b.append('\n'); - - if (this.selectedInUse) { - for (int j = 0; j < size; j++) { - int i = selected[j]; - b.append('['); - for (int k = 0; k < projectionSize; k++) { - int projIndex = projectedColumns[k]; - ColumnVector cv = cols[projIndex]; - if (k > 0) { - b.append(", "); - } - cv.stringifyValue(b, i); - } - b.append(']'); - if (j < size - 1) { - b.append('\n'); - } - } - } else { - for (int i = 0; i < size; i++) { - b.append('['); - for (int k = 0; k < projectionSize; k++) { - int projIndex = projectedColumns[k]; - ColumnVector cv = cols[projIndex]; - if (k > 0) { - b.append(", "); - } - if (cv != null) { - try { - cv.stringifyValue(b, i); - } catch (Exception ex) { - b.append(""); - } - } - } - b.append(']'); - if (i < size - 1) { - b.append('\n'); - } - } - } - return b.toString(); - } - - @Override - public void readFields(DataInput arg0) throws IOException { - throw new UnsupportedOperationException("Do you really need me?"); - } - - @Override - public void write(DataOutput arg0) throws IOException { - throw new UnsupportedOperationException("Don't call me"); - } - - /** - * Resets the row batch to default state - * - sets selectedInUse to false - * - sets size to 0 - * - sets endOfFile to false - * - resets each column - * - inits each column - */ - public void reset() { - selectedInUse = false; - size = 0; - endOfFile = false; - for (ColumnVector vc : cols) { - if (vc != null) { - vc.reset(); - vc.init(); - } - } - } - - /** - * Set the maximum number of rows in the batch. - * Data is not preserved. - */ - public void ensureSize(int rows) { - for(int i=0; i < cols.length; ++i) { - cols[i].ensureSize(rows, false); - } - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java deleted file mode 100644 index f2ae9bce709850345307c2068c49bfeb38dd9c44..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java +++ /dev/null @@ -1,414 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import java.util.Arrays; - -import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; - -/** - * String expression evaluation helper functions. - */ -public class StringExpr { - - /* Compare two strings from two byte arrays each - * with their own start position and length. - * Use lexicographic unsigned byte value order. - * This is what's used for UTF-8 sort order. - * Return negative value if arg1 < arg2, 0 if arg1 = arg2, - * positive if arg1 > arg2. - */ - public static int compare(byte[] arg1, int start1, int len1, byte[] arg2, int start2, int len2) { - for (int i = 0; i < len1 && i < len2; i++) { - // Note the "& 0xff" is just a way to convert unsigned bytes to signed integer. - int b1 = arg1[i + start1] & 0xff; - int b2 = arg2[i + start2] & 0xff; - if (b1 != b2) { - return b1 - b2; - } - } - return len1 - len2; - } - - /* Determine if two strings are equal from two byte arrays each - * with their own start position and length. - * Use lexicographic unsigned byte value order. - * This is what's used for UTF-8 sort order. - */ - public static boolean equal(byte[] arg1, final int start1, final int len1, - byte[] arg2, final int start2, final int len2) { - if (len1 != len2) { - return false; - } - if (len1 == 0) { - return true; - } - - // do bounds check for OOB exception - if (arg1[start1] != arg2[start2] - || arg1[start1 + len1 - 1] != arg2[start2 + len2 - 1]) { - return false; - } - - if (len1 == len2) { - // prove invariant to the compiler: len1 = len2 - // all array access between (start1, start1+len1) - // and (start2, start2+len2) are valid - // no more OOB exceptions are possible - final int step = 8; - final int remainder = len1 % step; - final int wlen = len1 - remainder; - // suffix first - for (int i = wlen; i < len1; i++) { - if (arg1[start1 + i] != arg2[start2 + i]) { - return false; - } - } - // SIMD loop - for (int i = 0; i < wlen; i += step) { - final int s1 = start1 + i; - final int s2 = start2 + i; - boolean neq = false; - for (int j = 0; j < step; j++) { - neq = (arg1[s1 + j] != arg2[s2 + j]) || neq; - } - if (neq) { - return false; - } - } - } - - return true; - } - - public static int characterCount(byte[] bytes) { - int end = bytes.length; - - // count characters - int j = 0; - int charCount = 0; - while(j < end) { - // UTF-8 continuation bytes have 2 high bits equal to 0x80. - if ((bytes[j] & 0xc0) != 0x80) { - ++charCount; - } - j++; - } - return charCount; - } - - public static int characterCount(byte[] bytes, int start, int length) { - int end = start + length; - - // count characters - int j = start; - int charCount = 0; - while(j < end) { - // UTF-8 continuation bytes have 2 high bits equal to 0x80. - if ((bytes[j] & 0xc0) != 0x80) { - ++charCount; - } - j++; - } - return charCount; - } - - // A setVal with the same function signature as rightTrim, leftTrim, truncate, etc, below. - // Useful for class generation via templates. - public static void assign(BytesColumnVector outV, int i, byte[] bytes, int start, int length) { - // set output vector - outV.setVal(i, bytes, start, length); - } - - /* - * Right trim a slice of a byte array and return the new byte length. - */ - public static int rightTrim(byte[] bytes, int start, int length) { - // skip trailing blank characters - int j = start + length - 1; - while(j >= start && bytes[j] == 0x20) { - j--; - } - - return (j - start) + 1; - } - - /* - * Right trim a slice of a byte array and place the result into element i of a vector. - */ - public static void rightTrim(BytesColumnVector outV, int i, byte[] bytes, int start, int length) { - // skip trailing blank characters - int j = start + length - 1; - while(j >= start && bytes[j] == 0x20) { - j--; - } - - // set output vector - outV.setVal(i, bytes, start, (j - start) + 1); - } - - /* - * Truncate a slice of a byte array to a maximum number of characters and - * return the new byte length. - */ - public static int truncate(byte[] bytes, int start, int length, int maxLength) { - int end = start + length; - - // count characters forward - int j = start; - int charCount = 0; - while(j < end) { - // UTF-8 continuation bytes have 2 high bits equal to 0x80. - if ((bytes[j] & 0xc0) != 0x80) { - if (charCount == maxLength) { - break; - } - ++charCount; - } - j++; - } - return (j - start); - } - - /* - * Truncate a slice of a byte array to a maximum number of characters and - * place the result into element i of a vector. - */ - public static void truncate(BytesColumnVector outV, int i, byte[] bytes, int start, int length, int maxLength) { - int end = start + length; - - // count characters forward - int j = start; - int charCount = 0; - while(j < end) { - // UTF-8 continuation bytes have 2 high bits equal to 0x80. - if ((bytes[j] & 0xc0) != 0x80) { - if (charCount == maxLength) { - break; - } - ++charCount; - } - j++; - } - - // set output vector - outV.setVal(i, bytes, start, (j - start)); - } - - /* - * Truncate a byte array to a maximum number of characters and - * return a byte array with only truncated bytes. - */ - public static byte[] truncateScalar(byte[] bytes, int maxLength) { - int end = bytes.length; - - // count characters forward - int j = 0; - int charCount = 0; - while(j < end) { - // UTF-8 continuation bytes have 2 high bits equal to 0x80. - if ((bytes[j] & 0xc0) != 0x80) { - if (charCount == maxLength) { - break; - } - ++charCount; - } - j++; - } - if (j == end) { - return bytes; - } else { - return Arrays.copyOf(bytes, j); - } - } - - /* - * Right trim and truncate a slice of a byte array to a maximum number of characters and - * return the new byte length. - */ - public static int rightTrimAndTruncate(byte[] bytes, int start, int length, int maxLength) { - int end = start + length; - - // count characters forward and watch for final run of pads - int j = start; - int charCount = 0; - int padRunStart = -1; - while(j < end) { - // UTF-8 continuation bytes have 2 high bits equal to 0x80. - if ((bytes[j] & 0xc0) != 0x80) { - if (charCount == maxLength) { - break; - } - if (bytes[j] == 0x20) { - if (padRunStart == -1) { - padRunStart = j; - } - } else { - padRunStart = -1; - } - ++charCount; - } else { - padRunStart = -1; - } - j++; - } - if (padRunStart != -1) { - return (padRunStart - start); - } else { - return (j - start); - } - } - - /* - * Right trim and truncate a slice of a byte array to a maximum number of characters and - * place the result into element i of a vector. - */ - public static void rightTrimAndTruncate(BytesColumnVector outV, int i, byte[] bytes, int start, int length, int maxLength) { - int end = start + length; - - // count characters forward and watch for final run of pads - int j = start; - int charCount = 0; - int padRunStart = -1; - while(j < end) { - // UTF-8 continuation bytes have 2 high bits equal to 0x80. - if ((bytes[j] & 0xc0) != 0x80) { - if (charCount == maxLength) { - break; - } - if (bytes[j] == 0x20) { - if (padRunStart == -1) { - padRunStart = j; - } - } else { - padRunStart = -1; - } - ++charCount; - } else { - padRunStart = -1; - } - j++; - } - // set output vector - if (padRunStart != -1) { - outV.setVal(i, bytes, start, (padRunStart - start)); - } else { - outV.setVal(i, bytes, start, (j - start) ); - } - } - - /* - * Right trim and truncate a byte array to a maximum number of characters and - * return a byte array with only the trimmed and truncated bytes. - */ - public static byte[] rightTrimAndTruncateScalar(byte[] bytes, int maxLength) { - int end = bytes.length; - - // count characters forward and watch for final run of pads - int j = 0; - int charCount = 0; - int padRunStart = -1; - while(j < end) { - // UTF-8 continuation bytes have 2 high bits equal to 0x80. - if ((bytes[j] & 0xc0) != 0x80) { - if (charCount == maxLength) { - break; - } - if (bytes[j] == 0x20) { - if (padRunStart == -1) { - padRunStart = j; - } - } else { - padRunStart = -1; - } - ++charCount; - } else { - padRunStart = -1; - } - j++; - } - if (padRunStart != -1) { - return Arrays.copyOf(bytes, padRunStart); - } else if (j == end) { - return bytes; - } else { - return Arrays.copyOf(bytes, j); - } - } - - /* - * Compiles the given pattern with a proper algorithm. - */ - public static Finder compile(byte[] pattern) { - return new BoyerMooreHorspool(pattern); - } - - /* - * A finder finds the first index of its pattern in a given byte array. - * Its thread-safety depends on its implementation. - */ - public interface Finder { - int find(byte[] input, int start, int len); - } - - /* - * StringExpr uses Boyer Moore Horspool algorithm to find faster. - * It is thread-safe, because it holds final member instances only. - * See https://en.wikipedia.org/wiki/Boyer–Moore–Horspool_algorithm . - */ - private static class BoyerMooreHorspool implements Finder { - private static final int MAX_BYTE = 0xff; - private final long[] shift = new long[MAX_BYTE]; - private final byte[] pattern; - private final int plen; - - public BoyerMooreHorspool(byte[] pattern) { - this.pattern = pattern; - this.plen = pattern.length; - Arrays.fill(shift, plen); - for (int i = 0; i < plen - 1; i++) { - shift[pattern[i] & MAX_BYTE] = plen - i - 1; - } - } - - public int find(byte[] input, int start, int len) { - if (pattern.length == 0) { - return 0; - } - - final int end = start + len; - int next = start + plen - 1; - final int plen = this.plen; - final byte[] pattern = this.pattern; - while (next < end) { - int s_tmp = next; - int p_tmp = plen - 1; - while (input[s_tmp] == pattern[p_tmp]) { - p_tmp--; - if (p_tmp < 0) { - return s_tmp; - } - s_tmp--; - } - next += shift[input[next] & MAX_BYTE]; - } - return -1; - } - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/ExpressionTree.java b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/ExpressionTree.java deleted file mode 100644 index 443083d228502f7c78f3bd4ece320c38812a6c68..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/ExpressionTree.java +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.io.sarg; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * The inner representation of the SearchArgument. Most users should not - * need this interface, it is only for file formats that need to translate - * the SearchArgument into an internal form. - */ -public class ExpressionTree { - public enum Operator {OR, AND, NOT, LEAF, CONSTANT} - private final Operator operator; - private final List children; - private int leaf; - private final SearchArgument.TruthValue constant; - - ExpressionTree() { - operator = null; - children = null; - leaf = 0; - constant = null; - } - - ExpressionTree(Operator op, ExpressionTree... kids) { - operator = op; - children = new ArrayList(); - leaf = -1; - this.constant = null; - Collections.addAll(children, kids); - } - - ExpressionTree(int leaf) { - operator = Operator.LEAF; - children = null; - this.leaf = leaf; - this.constant = null; - } - - ExpressionTree(SearchArgument.TruthValue constant) { - operator = Operator.CONSTANT; - children = null; - this.leaf = -1; - this.constant = constant; - } - - ExpressionTree(ExpressionTree other) { - this.operator = other.operator; - if (other.children == null) { - this.children = null; - } else { - this.children = new ArrayList(); - for(ExpressionTree child: other.children) { - children.add(new ExpressionTree(child)); - } - } - this.leaf = other.leaf; - this.constant = other.constant; - } - - public SearchArgument.TruthValue evaluate(SearchArgument.TruthValue[] leaves - ) { - SearchArgument.TruthValue result = null; - switch (operator) { - case OR: - for(ExpressionTree child: children) { - result = child.evaluate(leaves).or(result); - } - return result; - case AND: - for(ExpressionTree child: children) { - result = child.evaluate(leaves).and(result); - } - return result; - case NOT: - return children.get(0).evaluate(leaves).not(); - case LEAF: - return leaves[leaf]; - case CONSTANT: - return constant; - default: - throw new IllegalStateException("Unknown operator: " + operator); - } - } - - @Override - public String toString() { - StringBuilder buffer = new StringBuilder(); - switch (operator) { - case OR: - buffer.append("(or"); - for(ExpressionTree child: children) { - buffer.append(' '); - buffer.append(child.toString()); - } - buffer.append(')'); - break; - case AND: - buffer.append("(and"); - for(ExpressionTree child: children) { - buffer.append(' '); - buffer.append(child.toString()); - } - buffer.append(')'); - break; - case NOT: - buffer.append("(not "); - buffer.append(children.get(0)); - buffer.append(')'); - break; - case LEAF: - buffer.append("leaf-"); - buffer.append(leaf); - break; - case CONSTANT: - buffer.append(constant); - break; - } - return buffer.toString(); - } - - public Operator getOperator() { - return operator; - } - - public List getChildren() { - return children; - } - - public SearchArgument.TruthValue getConstant() { - return constant; - } - - public int getLeaf() { - return leaf; - } - - public void setLeaf(int leaf) { - this.leaf = leaf; - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/LiteralDelegate.java b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/LiteralDelegate.java deleted file mode 100644 index bd8a5ce57860d20f08c9012a2332c7dcdec9fee8..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/LiteralDelegate.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.io.sarg; - -import org.apache.hadoop.conf.Configurable; - -/** - * Interface to retrieve a literal value - */ -public interface LiteralDelegate extends Configurable { - - Object getLiteral(); - - String getId(); -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java deleted file mode 100644 index 469a3dad47c79e51d24115db469647596e7b9785..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.io.sarg; - -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; - -import java.sql.Date; -import java.sql.Timestamp; -import java.util.List; - -/** - * The primitive predicates that form a SearchArgument. - */ -public interface PredicateLeaf { - - /** - * The possible operators for predicates. To get the opposites, construct - * an expression with a not operator. - */ - public static enum Operator { - EQUALS, - NULL_SAFE_EQUALS, - LESS_THAN, - LESS_THAN_EQUALS, - IN, - BETWEEN, - IS_NULL - } - - /** - * The possible types for sargs. - */ - public static enum Type { - LONG(Long.class), // all of the integer types - FLOAT(Double.class), // float and double - STRING(String.class), // string, char, varchar - DATE(Date.class), - DECIMAL(HiveDecimalWritable.class), - TIMESTAMP(Timestamp.class), - BOOLEAN(Boolean.class); - - private final Class cls; - Type(Class cls) { - this.cls = cls; - } - - /** - * For all SARG leaves, the values must be the matching class. - * @return the value class - */ - public Class getValueClass() { - return cls; - } - } - - /** - * Get the operator for the leaf. - */ - public Operator getOperator(); - - /** - * Get the type of the column and literal by the file format. - */ - public Type getType(); - - /** - * Get the simple column name. - * @return the column name - */ - public String getColumnName(); - - /** - * Get the literal half of the predicate leaf. Adapt the original type for what orc needs - * - * @return an Integer, Long, Double, or String - */ - public Object getLiteral(); - - /** - * For operators with multiple literals (IN and BETWEEN), get the literals. - * - * @return the list of literals (Integer, Longs, Doubles, or Strings) - * - */ - public List getLiteralList(); -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgument.java b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgument.java deleted file mode 100644 index d70b3b067b54697ee564be6a0189971c20153146..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgument.java +++ /dev/null @@ -1,287 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.io.sarg; - -import java.util.List; - -/** - * Primary interface for - * SearchArgument, which are the subset of predicates - * that can be pushed down to the RecordReader. Each SearchArgument consists - * of a series of SearchClauses that must each be true for the row to be - * accepted by the filter. - * - * This requires that the filter be normalized into conjunctive normal form - * (CNF). - */ -public interface SearchArgument { - - /** - * The potential result sets of logical operations. - */ - public static enum TruthValue { - YES, NO, NULL, YES_NULL, NO_NULL, YES_NO, YES_NO_NULL; - - /** - * Compute logical or between the two values. - * @param right the other argument or null - * @return the result - */ - public TruthValue or(TruthValue right) { - if (right == null || right == this) { - return this; - } - if (right == YES || this == YES) { - return YES; - } - if (right == YES_NULL || this == YES_NULL) { - return YES_NULL; - } - if (right == NO) { - return this; - } - if (this == NO) { - return right; - } - if (this == NULL) { - if (right == NO_NULL) { - return NULL; - } else { - return YES_NULL; - } - } - if (right == NULL) { - if (this == NO_NULL) { - return NULL; - } else { - return YES_NULL; - } - } - return YES_NO_NULL; - } - - /** - * Compute logical AND between the two values. - * @param right the other argument or null - * @return the result - */ - public TruthValue and(TruthValue right) { - if (right == null || right == this) { - return this; - } - if (right == NO || this == NO) { - return NO; - } - if (right == NO_NULL || this == NO_NULL) { - return NO_NULL; - } - if (right == YES) { - return this; - } - if (this == YES) { - return right; - } - if (this == NULL) { - if (right == YES_NULL) { - return NULL; - } else { - return NO_NULL; - } - } - if (right == NULL) { - if (this == YES_NULL) { - return NULL; - } else { - return NO_NULL; - } - } - return YES_NO_NULL; - } - - public TruthValue not() { - switch (this) { - case NO: - return YES; - case YES: - return NO; - case NULL: - case YES_NO: - case YES_NO_NULL: - return this; - case NO_NULL: - return YES_NULL; - case YES_NULL: - return NO_NULL; - default: - throw new IllegalArgumentException("Unknown value: " + this); - } - } - - /** - * Does the RecordReader need to include this set of records? - * @return true unless none of the rows qualify - */ - public boolean isNeeded() { - switch (this) { - case NO: - case NULL: - case NO_NULL: - return false; - default: - return true; - } - } - } - - /** - * Get the leaf predicates that are required to evaluate the predicate. The - * list will have the duplicates removed. - * @return the list of leaf predicates - */ - public List getLeaves(); - - /** - * Get the expression tree. This should only needed for file formats that - * need to translate the expression to an internal form. - */ - public ExpressionTree getExpression(); - - /** - * Evaluate the entire predicate based on the values for the leaf predicates. - * @param leaves the value of each leaf predicate - * @return the value of hte entire predicate - */ - public TruthValue evaluate(TruthValue[] leaves); - - /** - * A builder object for contexts outside of Hive where it isn't easy to - * get a ExprNodeDesc. The user must call startOr, startAnd, or startNot - * before adding any leaves. - */ - public interface Builder { - - /** - * Start building an or operation and push it on the stack. - * @return this - */ - public Builder startOr(); - - /** - * Start building an and operation and push it on the stack. - * @return this - */ - public Builder startAnd(); - - /** - * Start building a not operation and push it on the stack. - * @return this - */ - public Builder startNot(); - - /** - * Finish the current operation and pop it off of the stack. Each start - * call must have a matching end. - * @return this - */ - public Builder end(); - - /** - * Add a less than leaf to the current item on the stack. - * @param column the name of the column - * @param type the type of the expression - * @param literal the literal - * @return this - */ - public Builder lessThan(String column, PredicateLeaf.Type type, - Object literal); - - /** - * Add a less than equals leaf to the current item on the stack. - * @param column the name of the column - * @param type the type of the expression - * @param literal the literal - * @return this - */ - public Builder lessThanEquals(String column, PredicateLeaf.Type type, - Object literal); - - /** - * Add an equals leaf to the current item on the stack. - * @param column the name of the column - * @param type the type of the expression - * @param literal the literal - * @return this - */ - public Builder equals(String column, PredicateLeaf.Type type, - Object literal); - - /** - * Add a null safe equals leaf to the current item on the stack. - * @param column the name of the column - * @param type the type of the expression - * @param literal the literal - * @return this - */ - public Builder nullSafeEquals(String column, PredicateLeaf.Type type, - Object literal); - - /** - * Add an in leaf to the current item on the stack. - * @param column the name of the column - * @param type the type of the expression - * @param literal the literal - * @return this - */ - public Builder in(String column, PredicateLeaf.Type type, - Object... literal); - - /** - * Add an is null leaf to the current item on the stack. - * @param column the name of the column - * @param type the type of the expression - * @return this - */ - public Builder isNull(String column, PredicateLeaf.Type type); - - /** - * Add a between leaf to the current item on the stack. - * @param column the name of the column - * @param type the type of the expression - * @param lower the literal - * @param upper the literal - * @return this - */ - public Builder between(String column, PredicateLeaf.Type type, - Object lower, Object upper); - - /** - * Add a truth value to the expression. - * @param truth - * @return this - */ - public Builder literal(TruthValue truth); - - /** - * Build and return the SearchArgument that has been defined. All of the - * starts must have been ended before this call. - * @return the new SearchArgument - */ - public SearchArgument build(); - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentFactory.java b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentFactory.java deleted file mode 100644 index 3c10c83644587def21f8f5d7677ccf9795f75426..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentFactory.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.io.sarg; - -import org.apache.hadoop.conf.Configuration; - -/** - * A factory for creating SearchArguments, as well as modifying those created by this factory. - */ -public class SearchArgumentFactory { - public static SearchArgument.Builder newBuilder() { - return newBuilder(null); - } - - public static SearchArgument.Builder newBuilder(Configuration conf) { - return new SearchArgumentImpl.BuilderImpl(conf); - } - - public static void setPredicateLeafColumn(PredicateLeaf leaf, String newName) { - SearchArgumentImpl.PredicateLeafImpl.setColumnName(leaf, newName); - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java deleted file mode 100644 index 85273f9c161f5db190987f63cb9437aa6bd8b8a6..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java +++ /dev/null @@ -1,739 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.io.sarg; - -import java.sql.Timestamp; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Deque; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.Set; - -import org.apache.hadoop.conf.Configuration; - -/** - * The implementation of SearchArguments. Visible for testing only. - */ -public final class SearchArgumentImpl implements SearchArgument { - - public static final class PredicateLeafImpl implements PredicateLeaf { - private final Operator operator; - private final Type type; - private String columnName; - private final Object literal; - private final List literalList; - - // Used by kryo - @SuppressWarnings("unused") - PredicateLeafImpl() { - operator = null; - type = null; - columnName = null; - literal = null; - literalList = null; - } - - public PredicateLeafImpl(Operator operator, - Type type, - String columnName, - Object literal, - List literalList) { - this(operator, type, columnName, literal, literalList, null); - } - - public PredicateLeafImpl(Operator operator, - Type type, - String columnName, - Object literal, - List literalList, Configuration conf) { - this.operator = operator; - this.type = type; - this.columnName = columnName; - this.literal = literal; - checkLiteralType(literal, type, conf); - this.literalList = literalList; - if (literalList != null) { - Class valueCls = type.getValueClass(); - for(Object lit: literalList) { - checkLiteralType(lit, type, conf); - } - } - } - - @Override - public Operator getOperator() { - return operator; - } - - @Override - public Type getType(){ - return type; - } - - @Override - public String getColumnName() { - return columnName; - } - - @Override - public Object getLiteral() { - if (literal instanceof LiteralDelegate) { - return ((LiteralDelegate) literal).getLiteral(); - } - - // To get around a kryo 2.22 bug while deserialize a Timestamp into Date - // (https://github.com/EsotericSoftware/kryo/issues/88) - // When we see a Date, convert back into Timestamp - if (literal instanceof java.util.Date) { - return new Timestamp(((java.util.Date)literal).getTime()); - } - return literal; - } - - @Override - public List getLiteralList() { - if (literalList != null && literalList.size() > 0 && literalList.get(0) instanceof LiteralDelegate) { - List newLiteraList = new ArrayList(); - for (Object litertalObj : literalList) { - Object literal = ((LiteralDelegate) litertalObj).getLiteral(); - if (literal != null) { - newLiteraList.add(literal); - } - } - return newLiteraList; - } - return literalList; - } - - @Override - public String toString() { - StringBuilder buffer = new StringBuilder(); - buffer.append('('); - buffer.append(operator); - buffer.append(' '); - buffer.append(columnName); - if (literal != null) { - buffer.append(' '); - buffer.append(literal); - } else if (literalList != null) { - for(Object lit: literalList) { - buffer.append(' '); - buffer.append(lit == null ? "null" : lit.toString()); - } - } - buffer.append(')'); - return buffer.toString(); - } - - private static boolean isEqual(Object left, Object right) { - - return left == right || - (left != null && right != null && left.equals(right)); - } - - @Override - public boolean equals(Object other) { - if (other == null || other.getClass() != getClass()) { - return false; - } else if (other == this) { - return true; - } else { - PredicateLeafImpl o = (PredicateLeafImpl) other; - return operator == o.operator && - type == o.type && - columnName.equals(o.columnName) && - isEqual(literal, o.literal) && - isEqual(literalList, o.literalList); - } - } - - @Override - public int hashCode() { - return operator.hashCode() + - type.hashCode() * 17 + - columnName.hashCode() * 3 * 17+ - (literal == null ? 0 : literal.hashCode()) * 101 * 3 * 17 + - (literalList == null ? 0 : literalList.hashCode()) * - 103 * 101 * 3 * 17; - } - - public static void setColumnName(PredicateLeaf leaf, String newName) { - assert leaf instanceof PredicateLeafImpl; - ((PredicateLeafImpl)leaf).columnName = newName; - } - - protected void checkLiteralType(Object literal, Type type, Configuration conf) { - if (literal == null) { - return; - } - - if (literal instanceof LiteralDelegate) { - // Give it a pass. Optionally, have LiteralDelegate provide a getLiteralClass() to check. - ((LiteralDelegate) literal).setConf(conf); - } else { - if (literal.getClass() != type.getValueClass()) { - throw new IllegalArgumentException("Wrong value class " + - literal.getClass().getName() + " for " + type + "." + operator + - " leaf"); - } - } - } - } - - private final List leaves; - private final ExpressionTree expression; - - SearchArgumentImpl(ExpressionTree expression, List leaves) { - this.expression = expression; - this.leaves = leaves; - } - - // Used by kyro - @SuppressWarnings("unused") - SearchArgumentImpl() { - leaves = null; - expression = null; - } - - @Override - public List getLeaves() { - return leaves; - } - - @Override - public TruthValue evaluate(TruthValue[] leaves) { - return expression == null ? TruthValue.YES : expression.evaluate(leaves); - } - - @Override - public ExpressionTree getExpression() { - return expression; - } - - @Override - public String toString() { - StringBuilder buffer = new StringBuilder(); - for(int i=0; i < leaves.size(); ++i) { - buffer.append("leaf-"); - buffer.append(i); - buffer.append(" = "); - buffer.append(leaves.get(i).toString()); - buffer.append(", "); - } - buffer.append("expr = "); - buffer.append(expression); - return buffer.toString(); - } - - static class BuilderImpl implements Builder { - - Configuration conf; - public BuilderImpl(Configuration conf) { - this.conf = conf; - } - - // max threshold for CNF conversion. having >8 elements in andList will be - // converted to maybe - private static final int CNF_COMBINATIONS_THRESHOLD = 256; - - private final Deque currentTree = - new ArrayDeque(); - private final Map leaves = - new HashMap(); - private final ExpressionTree root = - new ExpressionTree(ExpressionTree.Operator.AND); - { - currentTree.add(root); - } - - @Override - public Builder startOr() { - ExpressionTree node = new ExpressionTree(ExpressionTree.Operator.OR); - currentTree.getFirst().getChildren().add(node); - currentTree.addFirst(node); - return this; - } - - @Override - public Builder startAnd() { - ExpressionTree node = new ExpressionTree(ExpressionTree.Operator.AND); - currentTree.getFirst().getChildren().add(node); - currentTree.addFirst(node); - return this; - } - - @Override - public Builder startNot() { - ExpressionTree node = new ExpressionTree(ExpressionTree.Operator.NOT); - currentTree.getFirst().getChildren().add(node); - currentTree.addFirst(node); - return this; - } - - @Override - public Builder end() { - ExpressionTree current = currentTree.removeFirst(); - if (current.getChildren().size() == 0) { - throw new IllegalArgumentException("Can't create expression " + root + - " with no children."); - } - if (current.getOperator() == ExpressionTree.Operator.NOT && - current.getChildren().size() != 1) { - throw new IllegalArgumentException("Can't create not expression " + - current + " with more than 1 child."); - } - return this; - } - - private int addLeaf(PredicateLeaf leaf) { - Integer result = leaves.get(leaf); - if (result == null) { - int id = leaves.size(); - leaves.put(leaf, id); - return id; - } else { - return result; - } - } - - @Override - public Builder lessThan(String column, PredicateLeaf.Type type, - Object literal) { - ExpressionTree parent = currentTree.getFirst(); - if (column == null || literal == null) { - parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL)); - } else { - PredicateLeaf leaf = - new PredicateLeafImpl(PredicateLeaf.Operator.LESS_THAN, - type, column, literal, null, conf); - parent.getChildren().add(new ExpressionTree(addLeaf(leaf))); - } - return this; - } - - @Override - public Builder lessThanEquals(String column, PredicateLeaf.Type type, - Object literal) { - ExpressionTree parent = currentTree.getFirst(); - if (column == null || literal == null) { - parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL)); - } else { - PredicateLeaf leaf = - new PredicateLeafImpl(PredicateLeaf.Operator.LESS_THAN_EQUALS, - type, column, literal, null, conf); - parent.getChildren().add(new ExpressionTree(addLeaf(leaf))); - } - return this; - } - - @Override - public Builder equals(String column, PredicateLeaf.Type type, - Object literal) { - ExpressionTree parent = currentTree.getFirst(); - if (column == null || literal == null) { - parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL)); - } else { - PredicateLeaf leaf = - new PredicateLeafImpl(PredicateLeaf.Operator.EQUALS, - type, column, literal, null, conf); - parent.getChildren().add(new ExpressionTree(addLeaf(leaf))); - } - return this; - } - - @Override - public Builder nullSafeEquals(String column, PredicateLeaf.Type type, - Object literal) { - ExpressionTree parent = currentTree.getFirst(); - if (column == null || literal == null) { - parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL)); - } else { - PredicateLeaf leaf = - new PredicateLeafImpl(PredicateLeaf.Operator.NULL_SAFE_EQUALS, - type, column, literal, null, conf); - parent.getChildren().add(new ExpressionTree(addLeaf(leaf))); - } - return this; - } - - @Override - public Builder in(String column, PredicateLeaf.Type type, - Object... literal) { - ExpressionTree parent = currentTree.getFirst(); - if (column == null || literal == null) { - parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL)); - } else { - if (literal.length == 0) { - throw new IllegalArgumentException("Can't create in expression with " - + "no arguments"); - } - List argList = new ArrayList(); - argList.addAll(Arrays.asList(literal)); - - PredicateLeaf leaf = - new PredicateLeafImpl(PredicateLeaf.Operator.IN, - type, column, null, argList, conf); - parent.getChildren().add(new ExpressionTree(addLeaf(leaf))); - } - return this; - } - - @Override - public Builder isNull(String column, PredicateLeaf.Type type) { - ExpressionTree parent = currentTree.getFirst(); - if (column == null) { - parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL)); - } else { - PredicateLeaf leaf = - new PredicateLeafImpl(PredicateLeaf.Operator.IS_NULL, - type, column, null, null, conf); - parent.getChildren().add(new ExpressionTree(addLeaf(leaf))); - } - return this; - } - - @Override - public Builder between(String column, PredicateLeaf.Type type, Object lower, - Object upper) { - ExpressionTree parent = currentTree.getFirst(); - if (column == null || lower == null || upper == null) { - parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL)); - } else { - List argList = new ArrayList(); - argList.add(lower); - argList.add(upper); - PredicateLeaf leaf = - new PredicateLeafImpl(PredicateLeaf.Operator.BETWEEN, - type, column, null, argList, conf); - parent.getChildren().add(new ExpressionTree(addLeaf(leaf))); - } - return this; - } - - @Override - public Builder literal(TruthValue truth) { - ExpressionTree parent = currentTree.getFirst(); - parent.getChildren().add(new ExpressionTree(truth)); - return this; - } - - /** - * Recursively explore the tree to find the leaves that are still reachable - * after optimizations. - * @param tree the node to check next - * @param next the next available leaf id - * @param leafReorder - * @return the next available leaf id - */ - static int compactLeaves(ExpressionTree tree, int next, int[] leafReorder) { - if (tree.getOperator() == ExpressionTree.Operator.LEAF) { - int oldLeaf = tree.getLeaf(); - if (leafReorder[oldLeaf] == -1) { - leafReorder[oldLeaf] = next++; - } - } else if (tree.getChildren() != null){ - for(ExpressionTree child: tree.getChildren()) { - next = compactLeaves(child, next, leafReorder); - } - } - return next; - } - - /** - * Rewrite expression tree to update the leaves. - * @param root the root of the tree to fix - * @param leafReorder a map from old leaf ids to new leaf ids - * @return the fixed root - */ - static ExpressionTree rewriteLeaves(ExpressionTree root, - int[] leafReorder) { - // The leaves could be shared in the tree. Use Set to remove the duplicates. - Set leaves = new HashSet(); - Queue nodes = new LinkedList(); - nodes.add(root); - - while(!nodes.isEmpty()) { - ExpressionTree node = nodes.remove(); - if (node.getOperator() == ExpressionTree.Operator.LEAF) { - leaves.add(node); - } else { - if (node.getChildren() != null){ - nodes.addAll(node.getChildren()); - } - } - } - - // Update the leaf in place - for(ExpressionTree leaf : leaves) { - leaf.setLeaf(leafReorder[leaf.getLeaf()]); - } - - return root; - } - - @Override - public SearchArgument build() { - if (currentTree.size() != 1) { - throw new IllegalArgumentException("Failed to end " + - currentTree.size() + " operations."); - } - ExpressionTree optimized = pushDownNot(root); - optimized = foldMaybe(optimized); - optimized = flatten(optimized); - optimized = convertToCNF(optimized); - optimized = flatten(optimized); - int leafReorder[] = new int[leaves.size()]; - Arrays.fill(leafReorder, -1); - int newLeafCount = compactLeaves(optimized, 0, leafReorder); - optimized = rewriteLeaves(optimized, leafReorder); - ArrayList leafList = new ArrayList<>(newLeafCount); - // expand list to correct size - for(int i=0; i < newLeafCount; ++i) { - leafList.add(null); - } - // build the new list - for(Map.Entry elem: leaves.entrySet()) { - int newLoc = leafReorder[elem.getValue()]; - if (newLoc != -1) { - leafList.set(newLoc, elem.getKey()); - } - } - return new SearchArgumentImpl(optimized, leafList); - } - - /** - * Push the negations all the way to just before the leaves. Also remove - * double negatives. - * @param root the expression to normalize - * @return the normalized expression, which may share some or all of the - * nodes of the original expression. - */ - static ExpressionTree pushDownNot(ExpressionTree root) { - if (root.getOperator() == ExpressionTree.Operator.NOT) { - ExpressionTree child = root.getChildren().get(0); - switch (child.getOperator()) { - case NOT: - return pushDownNot(child.getChildren().get(0)); - case CONSTANT: - return new ExpressionTree(child.getConstant().not()); - case AND: - root = new ExpressionTree(ExpressionTree.Operator.OR); - for(ExpressionTree kid: child.getChildren()) { - root.getChildren().add(pushDownNot(new - ExpressionTree(ExpressionTree.Operator.NOT, kid))); - } - break; - case OR: - root = new ExpressionTree(ExpressionTree.Operator.AND); - for(ExpressionTree kid: child.getChildren()) { - root.getChildren().add(pushDownNot(new ExpressionTree - (ExpressionTree.Operator.NOT, kid))); - } - break; - // for leaf, we don't do anything - default: - break; - } - } else if (root.getChildren() != null) { - // iterate through children and push down not for each one - for(int i=0; i < root.getChildren().size(); ++i) { - root.getChildren().set(i, pushDownNot(root.getChildren().get(i))); - } - } - return root; - } - - /** - * Remove MAYBE values from the expression. If they are in an AND operator, - * they are dropped. If they are in an OR operator, they kill their parent. - * This assumes that pushDownNot has already been called. - * @param expr The expression to clean up - * @return The cleaned up expression - */ - static ExpressionTree foldMaybe(ExpressionTree expr) { - if (expr.getChildren() != null) { - for(int i=0; i < expr.getChildren().size(); ++i) { - ExpressionTree child = foldMaybe(expr.getChildren().get(i)); - if (child.getConstant() == TruthValue.YES_NO_NULL) { - switch (expr.getOperator()) { - case AND: - expr.getChildren().remove(i); - i -= 1; - break; - case OR: - // a maybe will kill the or condition - return child; - default: - throw new IllegalStateException("Got a maybe as child of " + - expr); - } - } else { - expr.getChildren().set(i, child); - } - } - if (expr.getChildren().isEmpty()) { - return new ExpressionTree(TruthValue.YES_NO_NULL); - } - } - return expr; - } - - /** - * Converts multi-level ands and ors into single level ones. - * @param root the expression to flatten - * @return the flattened expression, which will always be root with - * potentially modified children. - */ - static ExpressionTree flatten(ExpressionTree root) { - if (root.getChildren() != null) { - // iterate through the index, so that if we add more children, - // they don't get re-visited - for(int i=0; i < root.getChildren().size(); ++i) { - ExpressionTree child = flatten(root.getChildren().get(i)); - // do we need to flatten? - if (child.getOperator() == root.getOperator() && - child.getOperator() != ExpressionTree.Operator.NOT) { - boolean first = true; - for(ExpressionTree grandkid: child.getChildren()) { - // for the first grandkid replace the original parent - if (first) { - first = false; - root.getChildren().set(i, grandkid); - } else { - root.getChildren().add(++i, grandkid); - } - } - } else { - root.getChildren().set(i, child); - } - } - // if we have a singleton AND or OR, just return the child - if ((root.getOperator() == ExpressionTree.Operator.OR || - root.getOperator() == ExpressionTree.Operator.AND) && - root.getChildren().size() == 1) { - return root.getChildren().get(0); - } - } - return root; - } - - /** - * Generate all combinations of items on the andList. For each item on the - * andList, it generates all combinations of one child from each and - * expression. Thus, (and a b) (and c d) will be expanded to: (or a c) - * (or a d) (or b c) (or b d). If there are items on the nonAndList, they - * are added to each or expression. - * @param result a list to put the results onto - * @param andList a list of and expressions - * @param nonAndList a list of non-and expressions - */ - private static void generateAllCombinations(List result, - List andList, - List nonAndList - ) { - List kids = andList.get(0).getChildren(); - if (result.isEmpty()) { - for(ExpressionTree kid: kids) { - ExpressionTree or = new ExpressionTree(ExpressionTree.Operator.OR); - result.add(or); - for(ExpressionTree node: nonAndList) { - or.getChildren().add(new ExpressionTree(node)); - } - or.getChildren().add(kid); - } - } else { - List work = new ArrayList(result); - result.clear(); - for(ExpressionTree kid: kids) { - for(ExpressionTree or: work) { - ExpressionTree copy = new ExpressionTree(or); - copy.getChildren().add(kid); - result.add(copy); - } - } - } - if (andList.size() > 1) { - generateAllCombinations(result, andList.subList(1, andList.size()), - nonAndList); - } - } - - /** - * Convert an expression so that the top level operator is AND with OR - * operators under it. This routine assumes that all of the NOT operators - * have been pushed to the leaves via pushdDownNot. - * @param root the expression - * @return the normalized expression - */ - static ExpressionTree convertToCNF(ExpressionTree root) { - if (root.getChildren() != null) { - // convert all of the children to CNF - int size = root.getChildren().size(); - for(int i=0; i < size; ++i) { - root.getChildren().set(i, convertToCNF(root.getChildren().get(i))); - } - if (root.getOperator() == ExpressionTree.Operator.OR) { - // a list of leaves that weren't under AND expressions - List nonAndList = new ArrayList(); - // a list of AND expressions that we need to distribute - List andList = new ArrayList(); - for(ExpressionTree child: root.getChildren()) { - if (child.getOperator() == ExpressionTree.Operator.AND) { - andList.add(child); - } else if (child.getOperator() == ExpressionTree.Operator.OR) { - // pull apart the kids of the OR expression - for(ExpressionTree grandkid: child.getChildren()) { - nonAndList.add(grandkid); - } - } else { - nonAndList.add(child); - } - } - if (!andList.isEmpty()) { - if (checkCombinationsThreshold(andList)) { - root = new ExpressionTree(ExpressionTree.Operator.AND); - generateAllCombinations(root.getChildren(), andList, nonAndList); - } else { - root = new ExpressionTree(TruthValue.YES_NO_NULL); - } - } - } - } - return root; - } - - private static boolean checkCombinationsThreshold(List andList) { - int numComb = 1; - for (ExpressionTree tree : andList) { - numComb *= tree.getChildren().size(); - if (numComb > CNF_COMBINATIONS_THRESHOLD) { - return false; - } - } - return true; - } - - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java b/storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java deleted file mode 100644 index 68ea6dba41da74e324cc0cfb8026153062be2d3d..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java +++ /dev/null @@ -1,345 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.util; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Estimation of memory footprint of object - */ -public enum JavaDataModel { - - JAVA32 { - @Override - public int object() { - return JAVA32_OBJECT; - } - - @Override - public int array() { - return JAVA32_ARRAY; - } - - @Override - public int ref() { - return JAVA32_REF; - } - - @Override - public int hashMap(int entry) { - // base = JAVA32_OBJECT + PRIMITIVES1 * 4 + JAVA32_FIELDREF * 3 + JAVA32_ARRAY; - // entry = JAVA32_OBJECT + JAVA32_FIELDREF + PRIMITIVES1 - return hashMapBase() + hashMapEntry() * entry; - } - - @Override - public int hashMapBase() { - return 64; - } - - @Override - public int hashMapEntry() { - return 24; - } - - @Override - public int hashSet(int entry) { - // hashMap += JAVA32_OBJECT - return hashSetBase() + hashSetEntry() * entry; - } - - @Override - public int hashSetBase() { - return 80; - } - - @Override - public int hashSetEntry() { - return 24; - } - - @Override - public int linkedHashMap(int entry) { - // hashMap += JAVA32_FIELDREF + PRIMITIVES1 - // hashMap.entry += JAVA32_FIELDREF * 2 - return 72 + 32 * entry; - } - - @Override - public int linkedList(int entry) { - // base = JAVA32_OBJECT + PRIMITIVES1 * 2 + JAVA32_FIELDREF; - // entry = JAVA32_OBJECT + JAVA32_FIELDREF * 2 - return linkedListBase() + linkedListEntry() * entry; - } - - @Override - public int linkedListBase() { - return 28; - } - - @Override - public int linkedListEntry() { - return 24; - } - - @Override - public int arrayList() { - // JAVA32_OBJECT + PRIMITIVES1 * 2 + JAVA32_ARRAY; - return 44; - } - - @Override - public int memoryAlign() { - return 8; - } - }, JAVA64 { - @Override - public int object() { - return JAVA64_OBJECT; - } - - @Override - public int array() { - return JAVA64_ARRAY; - } - - @Override - public int ref() { - return JAVA64_REF; - } - - @Override - public int hashMap(int entry) { - // base = JAVA64_OBJECT + PRIMITIVES1 * 4 + JAVA64_FIELDREF * 3 + JAVA64_ARRAY; - // entry = JAVA64_OBJECT + JAVA64_FIELDREF + PRIMITIVES1 - return hashMapBase() + hashMapEntry() * entry; - } - - @Override - public int hashMapBase() { - return 112; - } - - - @Override - public int hashMapEntry() { - return 44; - } - - @Override - public int hashSet(int entry) { - // hashMap += JAVA64_OBJECT - return hashSetBase() + hashSetEntry() * entry; - } - - @Override - public int hashSetBase() { - return 144; - } - - @Override - public int hashSetEntry() { - return 44; - } - - @Override - public int linkedHashMap(int entry) { - // hashMap += JAVA64_FIELDREF + PRIMITIVES1 - // hashMap.entry += JAVA64_FIELDREF * 2 - return 128 + 60 * entry; - } - - @Override - public int linkedList(int entry) { - // base = JAVA64_OBJECT + PRIMITIVES1 * 2 + JAVA64_FIELDREF; - // entry = JAVA64_OBJECT + JAVA64_FIELDREF * 2 - return linkedListBase() + linkedListEntry() * entry; - } - - @Override - public int linkedListBase() { - return 48; - } - - @Override - public int linkedListEntry() { - return 48; - } - - @Override - public int arrayList() { - // JAVA64_OBJECT + PRIMITIVES1 * 2 + JAVA64_ARRAY; - return 80; - } - - @Override - public int memoryAlign() { - return 8; - } - }; - - public abstract int object(); - public abstract int array(); - public abstract int ref(); - public abstract int hashMap(int entry); - public abstract int hashMapBase(); - public abstract int hashMapEntry(); - public abstract int hashSetBase(); - public abstract int hashSetEntry(); - public abstract int hashSet(int entry); - public abstract int linkedHashMap(int entry); - public abstract int linkedListBase(); - public abstract int linkedListEntry(); - public abstract int linkedList(int entry); - public abstract int arrayList(); - public abstract int memoryAlign(); - - // ascii string - public long lengthFor(String string) { - return lengthForStringOfLength(string.length()); - } - - public int lengthForRandom() { - // boolean + double + AtomicLong - return object() + primitive1() + primitive2() + object() + primitive2(); - } - - public int primitive1() { - return PRIMITIVES1; - } - public int primitive2() { - return PRIMITIVES2; - } - - public static long alignUp(long value, long align) { - return (value + align - 1L) & ~(align - 1L); - } - - private static final Logger LOG = LoggerFactory.getLogger(JavaDataModel.class); - - public static final int JAVA32_META = 12; - public static final int JAVA32_ARRAY_META = 16; - public static final int JAVA32_REF = 4; - public static final int JAVA32_OBJECT = 16; // JAVA32_META + JAVA32_REF - public static final int JAVA32_ARRAY = 20; // JAVA32_ARRAY_META + JAVA32_REF - - public static final int JAVA64_META = 24; - public static final int JAVA64_ARRAY_META = 32; - public static final int JAVA64_REF = 8; - public static final int JAVA64_OBJECT = 32; // JAVA64_META + JAVA64_REF - public static final int JAVA64_ARRAY = 40; // JAVA64_ARRAY_META + JAVA64_REF - - public static final int PRIMITIVES1 = 4; // void, boolean, byte, short, int, float - public static final int PRIMITIVES2 = 8; // long, double - - public static final int PRIMITIVE_BYTE = 1; // byte - - private static final class LazyHolder { - private static final JavaDataModel MODEL_FOR_SYSTEM = getModelForSystem(); - } - - //@VisibleForTesting - static JavaDataModel getModelForSystem() { - String props = null; - try { - props = System.getProperty("sun.arch.data.model"); - } catch (Exception e) { - LOG.warn("Failed to determine java data model, defaulting to 64", e); - } - if ("32".equals(props)) { - return JAVA32; - } - // TODO: separate model is needed for compressedOops, which can be guessed from memory size. - return JAVA64; - } - - public static JavaDataModel get() { - return LazyHolder.MODEL_FOR_SYSTEM; - } - - public static int round(int size) { - JavaDataModel model = get(); - if (model == JAVA32 || size % 8 == 0) { - return size; - } - return ((size + 8) >> 3) << 3; - } - - public long lengthForPrimitiveArrayOfSize(int primitiveSize, long length) { - return alignUp(array() + primitiveSize*length, memoryAlign()); - } - - public long lengthForByteArrayOfSize(long length) { - return lengthForPrimitiveArrayOfSize(PRIMITIVE_BYTE, length); - } - public long lengthForObjectArrayOfSize(long length) { - return lengthForPrimitiveArrayOfSize(ref(), length); - } - public long lengthForLongArrayOfSize(long length) { - return lengthForPrimitiveArrayOfSize(primitive2(), length); - } - public long lengthForDoubleArrayOfSize(long length) { - return lengthForPrimitiveArrayOfSize(primitive2(), length); - } - public long lengthForIntArrayOfSize(long length) { - return lengthForPrimitiveArrayOfSize(primitive1(), length); - } - public long lengthForBooleanArrayOfSize(long length) { - return lengthForPrimitiveArrayOfSize(PRIMITIVE_BYTE, length); - } - public long lengthForTimestampArrayOfSize(long length) { - return lengthForPrimitiveArrayOfSize(lengthOfTimestamp(), length); - } - public long lengthForDateArrayOfSize(long length) { - return lengthForPrimitiveArrayOfSize(lengthOfDate(), length); - } - public long lengthForDecimalArrayOfSize(long length) { - return lengthForPrimitiveArrayOfSize(lengthOfDecimal(), length); - } - - public int lengthOfDecimal() { - // object overhead + 8 bytes for intCompact + 4 bytes for precision - // + 4 bytes for scale + size of BigInteger - return object() + 2 * primitive2() + lengthOfBigInteger(); - } - - private int lengthOfBigInteger() { - // object overhead + 4 bytes for bitCount + 4 bytes for bitLength - // + 4 bytes for firstNonzeroByteNum + 4 bytes for firstNonzeroIntNum + - // + 4 bytes for lowestSetBit + 5 bytes for size of magnitude (since max precision - // is only 38 for HiveDecimal) + 7 bytes of padding (since java memory allocations - // are 8 byte aligned) - return object() + 4 * primitive2(); - } - - public int lengthOfTimestamp() { - // object overhead + 4 bytes for int (nanos) + 4 bytes of padding - return object() + primitive2(); - } - - public int lengthOfDate() { - // object overhead + 8 bytes for long (fastTime) + 16 bytes for cdate - return object() + 3 * primitive2(); - } - - public int lengthForStringOfLength(int strLen) { - return object() + primitive1() * 3 + array() + strLen; - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java b/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java deleted file mode 100644 index dfc72728bf5a71bef3686880035a30843e8df23a..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.util; - -import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.common.type.HiveDecimalV1; -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; - -import java.math.BigDecimal; -import java.sql.Timestamp; - -/** - * Utitilities for Timestamps and the relevant conversions. - */ -public class TimestampUtils { - public static final BigDecimal BILLION_BIG_DECIMAL = BigDecimal.valueOf(1000000000); - - /** - * Convert the timestamp to a double measured in seconds. - * @return double representation of the timestamp, accurate to nanoseconds - */ - public static double getDouble(Timestamp ts) { - long seconds = millisToSeconds(ts.getTime()); - return seconds + ((double) ts.getNanos()) / 1000000000; - } - - public static Timestamp doubleToTimestamp(double f) { - try { - long seconds = (long) f; - - // We must ensure the exactness of the double's fractional portion. - // 0.6 as the fraction part will be converted to 0.59999... and - // significantly reduce the savings from binary serialization - BigDecimal bd = new BigDecimal(String.valueOf(f)); - - bd = bd.subtract(new BigDecimal(seconds)).multiply(new BigDecimal(1000000000)); - int nanos = bd.intValue(); - - // Convert to millis - long millis = seconds * 1000; - if (nanos < 0) { - millis -= 1000; - nanos += 1000000000; - } - Timestamp t = new Timestamp(millis); - - // Set remaining fractional portion to nanos - t.setNanos(nanos); - return t; - } catch (NumberFormatException nfe) { - return null; - } catch (IllegalArgumentException iae) { - return null; - } - } - - /** - * Take a HiveDecimal and return the timestamp representation where the fraction part is the - * nanoseconds and integer part is the number of seconds. - * @param dec - * @return - */ - public static Timestamp decimalToTimestamp(HiveDecimal dec) { - - HiveDecimalWritable nanosWritable = new HiveDecimalWritable(dec); - nanosWritable.mutateFractionPortion(); // Clip off seconds portion. - nanosWritable.mutateScaleByPowerOfTen(9); // Bring nanoseconds into integer portion. - if (!nanosWritable.isSet() || !nanosWritable.isInt()) { - return null; - } - int nanos = nanosWritable.intValue(); - if (nanos < 0) { - nanos += 1000000000; - } - nanosWritable.setFromLong(nanos); - - HiveDecimalWritable nanoInstant = new HiveDecimalWritable(dec); - nanoInstant.mutateScaleByPowerOfTen(9); - - nanoInstant.mutateSubtract(nanosWritable); - nanoInstant.mutateScaleByPowerOfTen(-9); // Back to seconds. - if (!nanoInstant.isSet() || !nanoInstant.isLong()) { - return null; - } - long seconds = nanoInstant.longValue(); - Timestamp t = new Timestamp(seconds * 1000); - t.setNanos(nanos); - return t; - } - - /** - * Take a HiveDecimalWritable and return the timestamp representation where the fraction part - * is the nanoseconds and integer part is the number of seconds. - * - * This is a HiveDecimalWritable variation with supplied scratch objects. - * @param decWritable - * @param scratchDecWritable1 - * @param scratchDecWritable2 - * @return - */ - public static Timestamp decimalToTimestamp( - HiveDecimalWritable decWritable, - HiveDecimalWritable scratchDecWritable1, HiveDecimalWritable scratchDecWritable2) { - - HiveDecimalWritable nanosWritable = scratchDecWritable1; - nanosWritable.set(decWritable); - nanosWritable.mutateFractionPortion(); // Clip off seconds portion. - nanosWritable.mutateScaleByPowerOfTen(9); // Bring nanoseconds into integer portion. - if (!nanosWritable.isSet() || !nanosWritable.isInt()) { - return null; - } - int nanos = nanosWritable.intValue(); - if (nanos < 0) { - nanos += 1000000000; - } - nanosWritable.setFromLong(nanos); - - HiveDecimalWritable nanoInstant = scratchDecWritable2; - nanoInstant.set(decWritable); - nanoInstant.mutateScaleByPowerOfTen(9); - - nanoInstant.mutateSubtract(nanosWritable); - nanoInstant.mutateScaleByPowerOfTen(-9); // Back to seconds. - if (!nanoInstant.isSet() || !nanoInstant.isLong()) { - return null; - } - long seconds = nanoInstant.longValue(); - - Timestamp timestamp = new Timestamp(seconds * 1000L); - timestamp.setNanos(nanos); - return timestamp; - } - - public static Timestamp decimalToTimestamp(HiveDecimalV1 dec) { - try { - BigDecimal nanoInstant = dec.bigDecimalValue().multiply(BILLION_BIG_DECIMAL); - int nanos = nanoInstant.remainder(BILLION_BIG_DECIMAL).intValue(); - if (nanos < 0) { - nanos += 1000000000; - } - long seconds = - nanoInstant.subtract(new BigDecimal(nanos)).divide(BILLION_BIG_DECIMAL).longValue(); - Timestamp t = new Timestamp(seconds * 1000); - t.setNanos(nanos); - - return t; - } catch (NumberFormatException nfe) { - return null; - } catch (IllegalArgumentException iae) { - return null; - } - } - - /** - * Rounds the number of milliseconds relative to the epoch down to the nearest whole number of - * seconds. 500 would round to 0, -500 would round to -1. - */ - public static long millisToSeconds(long millis) { - if (millis >= 0) { - return millis / 1000; - } else { - return (millis - 999) / 1000; - } - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java deleted file mode 100644 index 637720a35e830b26b1c1440a40bb9f9edc9005fe..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/DateWritable.java +++ /dev/null @@ -1,225 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.io; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.sql.Date; -import java.util.Calendar; -import java.util.GregorianCalendar; -import java.util.TimeZone; -import java.util.concurrent.TimeUnit; - -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.io.WritableUtils; - - -/** - * DateWritable - * Writable equivalent of java.sql.Date. - * - * Dates are of the format - * YYYY-MM-DD - * - */ -public class DateWritable implements WritableComparable { - - private static final long MILLIS_PER_DAY = TimeUnit.DAYS.toMillis(1); - - // Local time zone. Store separately because Calendar would clone it. - // Java TimeZone has no mention of thread safety. Use thread local instance to be safe. - private static final ThreadLocal LOCAL_TIMEZONE = new ThreadLocal() { - @Override - protected TimeZone initialValue() { - return Calendar.getInstance().getTimeZone(); - } - }; - - private static final ThreadLocal UTC_CALENDAR = new ThreadLocal() { - @Override - protected Calendar initialValue() { - return new GregorianCalendar(TimeZone.getTimeZone("UTC")); - } - }; - private static final ThreadLocal LOCAL_CALENDAR = new ThreadLocal() { - @Override - protected Calendar initialValue() { - return Calendar.getInstance(); - } - }; - - // Internal representation is an integer representing day offset from our epoch value 1970-01-01 - private int daysSinceEpoch = 0; - - /* Constructors */ - public DateWritable() { - } - - public DateWritable(DateWritable d) { - set(d); - } - - public DateWritable(Date d) { - set(d); - } - - public DateWritable(int d) { - set(d); - } - - /** - * Set the DateWritable based on the days since epoch date. - * @param d integer value representing days since epoch date - */ - public void set(int d) { - daysSinceEpoch = d; - } - - /** - * Set the DateWritable based on the year/month/day of the date in the local timezone. - * @param d Date value - */ - public void set(Date d) { - if (d == null) { - daysSinceEpoch = 0; - return; - } - - set(dateToDays(d)); - } - - public void set(DateWritable d) { - set(d.daysSinceEpoch); - } - - /** - * @return Date value corresponding to the date in the local time zone - */ - public Date get() { - return get(true); - } - - // TODO: we should call this more often. In theory, for DATE type, time should never matter, but - // it's hard to tell w/some code paths like UDFs/OIs etc. that are used in many places. - public Date get(boolean doesTimeMatter) { - return new Date(daysToMillis(daysSinceEpoch, doesTimeMatter)); - } - - public int getDays() { - return daysSinceEpoch; - } - - /** - * - * @return time in seconds corresponding to this DateWritable - */ - public long getTimeInSeconds() { - return get().getTime() / 1000; - } - - public static Date timeToDate(long l) { - return new Date(l * 1000); - } - - public static long daysToMillis(int d) { - return daysToMillis(d, true); - } - - public static long daysToMillis(int d, boolean doesTimeMatter) { - // What we are trying to get is the equivalent of new Date(ymd).getTime() in the local tz, - // where ymd is whatever d represents. How it "works" is this. - // First we get the UTC midnight for that day (which always exists, a small island of sanity). - long utcMidnight = d * MILLIS_PER_DAY; - // Now we take a local TZ offset at midnight UTC. Say we are in -4; that means (surprise - // surprise) that at midnight UTC it was 20:00 in local. So far we are on firm ground. - long utcMidnightOffset = LOCAL_TIMEZONE.get().getOffset(utcMidnight); - // And now we wander straight into the swamp, when instead of adding, we subtract it from UTC - // midnight to supposedly get local midnight (in the above case, 4:00 UTC). Of course, given - // all the insane DST variations, where we actually end up is anyone's guess. - long hopefullyMidnight = utcMidnight - utcMidnightOffset; - // Then we determine the local TZ offset at that magical time. - long offsetAtHM = LOCAL_TIMEZONE.get().getOffset(hopefullyMidnight); - // If the offsets are the same, we assume our initial jump did not cross any DST boundaries, - // and is thus valid. Both times flowed at the same pace. We congratulate ourselves and bail. - if (utcMidnightOffset == offsetAtHM) return hopefullyMidnight; - // Alas, we crossed some DST boundary. If the time of day doesn't matter to the caller, we'll - // simply get the next day and go back half a day. This is not ideal but seems to work. - if (!doesTimeMatter) return daysToMillis(d + 1) - (MILLIS_PER_DAY >> 1); - // Now, we could get previous and next day, figure our how many hours were inserted or removed, - // and from which of the days, etc. But at this point our gun is pointing straight at our foot, - // so let's just go the safe, expensive way. - Calendar utc = UTC_CALENDAR.get(), local = LOCAL_CALENDAR.get(); - utc.setTimeInMillis(utcMidnight); - local.set(utc.get(Calendar.YEAR), utc.get(Calendar.MONTH), utc.get(Calendar.DAY_OF_MONTH)); - return local.getTimeInMillis(); - } - - public static int millisToDays(long millisLocal) { - // We assume millisLocal is midnight of some date. What we are basically trying to do - // here is go from local-midnight to UTC-midnight (or whatever time that happens to be). - long millisUtc = millisLocal + LOCAL_TIMEZONE.get().getOffset(millisLocal); - int days; - if (millisUtc >= 0L) { - days = (int) (millisUtc / MILLIS_PER_DAY); - } else { - days = (int) ((millisUtc - 86399999 /*(MILLIS_PER_DAY - 1)*/) / MILLIS_PER_DAY); - } - return days; - } - - public static int dateToDays(Date d) { - // convert to equivalent time in UTC, then get day offset - long millisLocal = d.getTime(); - return millisToDays(millisLocal); - } - - @Override - public void readFields(DataInput in) throws IOException { - daysSinceEpoch = WritableUtils.readVInt(in); - } - - @Override - public void write(DataOutput out) throws IOException { - WritableUtils.writeVInt(out, daysSinceEpoch); - } - - @Override - public int compareTo(DateWritable d) { - return daysSinceEpoch - d.daysSinceEpoch; - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof DateWritable)) { - return false; - } - return compareTo((DateWritable) o) == 0; - } - - @Override - public String toString() { - // For toString, the time does not matter - return get(false).toString(); - } - - @Override - public int hashCode() { - return daysSinceEpoch; - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java deleted file mode 100644 index 7a3bf4d08f389cd814565672cf454816c716e464..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java +++ /dev/null @@ -1,1099 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.io; - -import java.util.Arrays; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.IOException; -import java.math.BigInteger; - -import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.common.type.FastHiveDecimal; -import org.apache.hadoop.hive.common.type.FastHiveDecimalImpl; -import org.apache.hadoop.hive.common.type.HiveDecimalVersionV2; -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.io.WritableUtils; - -/** - * A mutable decimal. - *

- * ------------------------------------- Version 2 ------------------------------------------------ - *

- * This is the 2nd major version of HiveDecimalWritable called V2. The previous version has been - * renamed to HiveDecimalWritableV1 and is kept as a test and behavior reference. - *

- * For good performance we do not represent the decimal using a byte array containing BigInteger - * bytes like the previous version V1 did. Instead V2 HiveDecimalWritable is is a private subclass - * of the same fast decimal class also used by HiveDecimal. So it stores the decimal value - * directly. - *

- * Many of the methods of HiveDecimal have been added to HiveDecimalWritable in V2 so code can - * modify the decimal instead of calling getHiveDecimal(), doing operations on HiveDecimal, and then - * setting HiveDecimalWritable back to the result. - *

- * Operations that modify have a prefix of "mutate" in their name. For example mutateAdd is used - * instead of the immutable operation add in HiveDecimal that returns a new decimal object. - *

- * This should have much better performance. - *

- * The original V1 public methods and fields are annotated with @HiveDecimalWritableVersionV1; new - * public methods and fields are annotated with @HiveDecimalWritableVersionV2. - * - */ -public final class HiveDecimalWritable extends FastHiveDecimal - implements WritableComparable { - - // Is the decimal value currently valid? - private boolean isSet; - - /* - * Scratch arrays used in fastBigIntegerBytes calls for better performance. - */ - - // An optional long array of length FastHiveDecimal.FAST_SCRATCH_LONGS_LEN. - private long[] internalScratchLongs; - - // An optional byte array of FastHiveDecimal.FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES. - private byte[] internalScratchBuffer; - - /** - * Create a decimal writable with no current value (i.e. isSet() will return false). - */ - @HiveDecimalWritableVersionV1 - public HiveDecimalWritable() { - super(); - isSet = false; - internalScratchLongs = null; - internalScratchBuffer = null; - } - - /** - * Create a decimal writable with an initial value from a String. - *

- * If the conversion from String to decimal is successful, isSet() will return true. - * - */ - @HiveDecimalWritableVersionV1 - public HiveDecimalWritable(String string) { - super(); - isSet = fastSetFromString(string, false); - if (!isSet) { - fastReset(); - } - } - - /** - * Create a decimal writable with an initial value from BigInteger bytes and a specified scale. - *

- * If the conversion to decimal is successful, isSet() will return true. - * - */ - @HiveDecimalWritableVersionV1 - public HiveDecimalWritable(byte[] bigIntegerBytes, int scale) { - super(); - setFromBigIntegerBytesAndScale(bigIntegerBytes, scale); - } - - /** - * Create a decimal writable with an initial value from another decimal writable. - *

- * If the supplied writable has a value, isSet() will return true. - * - */ - @HiveDecimalWritableVersionV1 - public HiveDecimalWritable(HiveDecimalWritable writable) { - super(); - set(writable); - } - - /** - * Create a decimal writable with an initial value from a HiveDecimal. - *

- * Afterwards, the isSet() method will return true, unless value is null. - * - */ - @HiveDecimalWritableVersionV1 - public HiveDecimalWritable(HiveDecimal value) { - super(); - set(value); - } - - /** - * Create a decimal writable with an initial value from a long with scale 0. - *

- * Afterwards, the isSet() method will return true. - * - */ - @HiveDecimalWritableVersionV1 - public HiveDecimalWritable(long longValue) { - super(); - setFromLong(longValue); - } - - /** - * Set the writable's current value to a HiveDecimal's value. - *

- * Afterwards, the isSet() method will return true, unless value is null. - * - */ - @HiveDecimalWritableVersionV1 - public void set(HiveDecimal value) { - if (value == null) { - fastReset(); - isSet = false; - } else { - fastSet(value); - isSet = true; - } - } - - /** - * Set the writable's current value to a HiveDecimal's value with a specified precision / scale - * enforced. - *

- * Afterwards, the isSet() method will return true, unless value is null or value didn't fit within - * maxPrecision / maxScale. - * - */ - @HiveDecimalWritableVersionV1 - public void set(HiveDecimal value, int maxPrecision, int maxScale) { - set(value); - if (isSet) { - isSet = fastEnforcePrecisionScale(maxPrecision, maxScale); - if (!isSet) { - fastReset(); - } - } - } - - /** - * Set the writable's current value to the value in a another decimal writable. - *

- * If the supplied writable has a value, isSet() will return true. - * - */ - @HiveDecimalWritableVersionV1 - public void set(HiveDecimalWritable writable) { - if (writable == null || !writable.isSet()) { - fastReset(); - isSet = false; - } else { - fastSet(writable); - isSet = true; - } - } - - /** - * Set a decimal writable's value from BigInteger bytes and a specified scale. - *

- * If the conversion to decimal is successful, isSet() will return true. - * - */ - @HiveDecimalWritableVersionV1 - public void set(byte[] bigIntegerBytes, int scale) { - setFromBigIntegerBytesAndScale(bigIntegerBytes, scale); - } - - /** - * Set the writable's current value to a writable's value with a specified precision / scale - * enforced. - *

- * The isSet() method will return true, unless value is null or value didn't fit within - * maxPrecision / maxScale. - * - */ - @HiveDecimalWritableVersionV2 - public void set(HiveDecimalWritable writable, int maxPrecision, int maxScale) { - set(writable); - if (isSet) { - isSet = fastEnforcePrecisionScale(maxPrecision, maxScale); - if (!isSet) { - fastReset(); - } - } - } - - /** - * Set a decimal writable's value to a long's value with scale 0. - *

- * Afterwards, the isSet() method will return true since all long values fit in a decimal. - * - */ - @HiveDecimalWritableVersionV2 - public void setFromLong(long longValue) { - fastReset(); - fastSetFromLong(longValue); - isSet = true; - } - - /** - * Set a decimal writable's value to a doubles value. - *

- * Afterwards, the isSet() method will return true if the double to decimal conversion was successful. - * - */ - @HiveDecimalWritableVersionV2 - public void setFromDouble(double doubleValue) { - fastReset(); - isSet = fastSetFromDouble(doubleValue); - if (!isSet) { - fastReset(); - } - } - - /** - * Set the writable's current value to the decimal in a UTF-8 byte slice. - *

- * Afterwards, the isSet() method will return true, unless byte slice could not be converted. - * - */ - @HiveDecimalWritableVersionV2 - public void setFromBytes(byte[] bytes, int offset, int length) { - fastReset(); - isSet = fastSetFromBytes(bytes, offset, length, false); - if (!isSet) { - fastReset(); - } - } - - @HiveDecimalWritableVersionV2 - public void setFromBytes(byte[] bytes, int offset, int length, boolean trimBlanks) { - fastReset(); - isSet = fastSetFromBytes(bytes, offset, length, trimBlanks); - if (!isSet) { - fastReset(); - } - } - - /** - * Set the writable's current value to the decimal digits only in a UTF-8 byte slice, a sign - * flag, and a scale. - *

- * Afterwards, the isSet() method will return true, unless byte slice etc could not be converted. - * - */ - @HiveDecimalWritableVersionV2 - public void setFromDigitsOnlyBytesWithScale( - boolean isNegative, byte[] bytes, int offset, int length, int scale) { - fastReset(); - isSet = fastSetFromDigitsOnlyBytesAndScale(isNegative, bytes, offset, length, scale); - if (!isSet) { - fastReset(); - } - } - - /** - * Set the writable's current value to the signed value from BigInteger bytes and a specified - * scale. - *

- * Afterwards, the isSet() method will return true, unless conversion failed. - * - */ - @HiveDecimalWritableVersionV2 - public void setFromBigIntegerBytesAndScale(byte[] bigIntegerBytes, int scale) { - fastReset(); - isSet = fastSetFromBigIntegerBytesAndScale(bigIntegerBytes, 0, bigIntegerBytes.length, scale); - if (!isSet) { - fastReset(); - } - } - - @HiveDecimalWritableVersionV2 - public void setFromBigIntegerBytesAndScale( - byte[] bigIntegerBytes, int offset, int length, int scale) { - fastReset(); - isSet = fastSetFromBigIntegerBytesAndScale(bigIntegerBytes, offset, length, scale); - if (!isSet) { - fastReset(); - } - } - - /** - * Set the writable's current value to the long's value at a specified - * scale. - *

- * Afterwards, the isSet() method will return true, unless conversion failed. - * - */ - @HiveDecimalWritableVersionV2 - public void setFromLongAndScale(long longValue, int scale) { - fastReset(); - isSet = fastSetFromLongAndScale(longValue, scale); - if (!isSet) { - fastReset(); - } - } - - /** - * Does this writable have a current value? - *

- * A return of false means a current value wasn't set, or an operation like mutateAdd overflowed, - * or a set* method couldn't convert the input value, etc. - * - */ - @HiveDecimalWritableVersionV2 - public boolean isSet() { - return isSet; - } - - /** - * Returns a HiveDecimal for the writable's current value. - *

- * Returns null if the writable isn't set. - * - */ - @HiveDecimalWritableVersionV1 - public HiveDecimal getHiveDecimal() { - if (!isSet) { - return null; - } - HiveDecimal result = HiveDecimal.createFromFast(this); - return result; - } - - /** - * Get a HiveDecimal instance from the writable and constraint it with maximum precision/scale. - *

- * @param maxPrecision maximum precision - * @param maxScale maximum scale - * @return HiveDecimal instance - */ - @HiveDecimalWritableVersionV1 - public HiveDecimal getHiveDecimal(int maxPrecision, int maxScale) { - if (!isSet) { - return null; - } - HiveDecimal dec = HiveDecimal.createFromFast(this); - HiveDecimal result = HiveDecimal.enforcePrecisionScale(dec, maxPrecision, maxScale); - return result; - } - - /** - * Standard Writable method that deserialize the fields of this object from a DataInput. - * - */ - @HiveDecimalWritableVersionV1 - @Override - public void readFields(DataInput in) throws IOException { - int scale = WritableUtils.readVInt(in); - int byteArrayLen = WritableUtils.readVInt(in); - byte[] bytes = new byte[byteArrayLen]; - in.readFully(bytes); - - fastReset(); - if (!fastSetFromBigIntegerBytesAndScale(bytes, 0, bytes.length, scale)) { - throw new IOException("Couldn't convert decimal"); - } - isSet = true; - } - - /** - * Standard Writable method that serialize the fields of this object to a DataOutput. - * - */ - @HiveDecimalWritableVersionV1 - @Override - public void write(DataOutput out) throws IOException { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - - if (internalScratchLongs == null) { - internalScratchLongs = new long[FastHiveDecimal.FAST_SCRATCH_LONGS_LEN]; - internalScratchBuffer = new byte[FastHiveDecimal.FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; - } - - write(out, internalScratchLongs, internalScratchBuffer); - } - - - /** - * A variation of the standard Writable method that serialize the fields of this object to a - * DataOutput with scratch buffers for good performance. - *

- * Allocate scratchLongs with HiveDecimal.SCRATCH_LONGS_LEN longs. - * And, allocate scratch buffer with HiveDecimal.SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes. - * - */ - @HiveDecimalWritableVersionV2 - public void write( - DataOutput out, - long[] scratchLongs, byte[] scratchBuffer) throws IOException { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - - WritableUtils.writeVInt(out, fastScale()); - - int byteLength = - fastBigIntegerBytes( - scratchLongs, scratchBuffer); - if (byteLength == 0) { - throw new RuntimeException("Couldn't convert decimal to binary"); - } - - WritableUtils.writeVInt(out, byteLength); - out.write(scratchBuffer, 0, byteLength); - } - - /** - * See the comments for HiveDecimal.serializationUtilsRead. - */ - @HiveDecimalWritableVersionV2 - public boolean serializationUtilsRead( - InputStream inputStream, int scale, - byte[] scratchBytes) - throws IOException { - fastReset(); - isSet = - fastSerializationUtilsRead( - inputStream, - scale, - scratchBytes); - return isSet; - } - - /** - * See the comments for HiveDecimal.serializationUtilsWrite. - */ - @HiveDecimalWritableVersionV2 - public boolean serializationUtilsWrite( - OutputStream outputStream, - long[] scratchLongs) - throws IOException { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return - fastSerializationUtilsWrite( - outputStream, - scratchLongs); - } - - /* - * Maximum number of decimal digits in a decimal64 long. - */ - @HiveDecimalWritableVersionV2 - public static final int DECIMAL64_DECIMAL_DIGITS = FastHiveDecimalImpl.DECIMAL64_DECIMAL_DIGITS; - - /* - * Test whether a precision will fit within a decimal64 (64-bit signed long with <= 18 decimal - * digits). - */ - @HiveDecimalWritableVersionV2 - public static boolean isPrecisionDecimal64(int precision) { - return (precision <= DECIMAL64_DECIMAL_DIGITS); - } - - /* - * Return the maximum absolute decimal64 value for a precision. - */ - @HiveDecimalWritableVersionV2 - public static long getDecimal64AbsMax(int precision) { - return FastHiveDecimalImpl.getDecimal64AbsMax(precision); - } - - /* - * Deserializes 64-bit decimals up to the maximum 64-bit precision (18 decimal digits). - * - * NOTE: Major assumption: the input decimal64 has already been bounds checked and a least - * has a precision <= DECIMAL64_DECIMAL_DIGITS. We do not bounds check here for better - * performance. You can bounds check beforehand with: - * Math.abs(decimal64Long) <= getDecimal64AbsMax(precision) - */ - @HiveDecimalWritableVersionV2 - public void deserialize64( - long decimal64Long, int scale) { - fastDeserialize64(decimal64Long, scale); - isSet = true; - } - - /* - * Serializes decimal64 up to the maximum 64-bit precision (18 decimal digits). - * - * NOTE: Major assumption: the fast decimal has already been bounds checked and a least - * has a precision <= DECIMAL64_DECIMAL_DIGITS. We do not bounds check here for better - * performance. - */ - @HiveDecimalWritableVersionV2 - public long serialize64(int scale) { - return fastSerialize64(scale); - } - - @HiveDecimalWritableVersionV2 - public boolean isValid() { - if (!isSet) { - return false; - } - return FastHiveDecimalImpl.fastIsValid(this); - } - - /** - * Returns the length of the decimal converted to bytes. - * Call bigIntegerBytesBuffer() to get a reference to the converted bytes. - * - */ - @HiveDecimalWritableVersionV2 - public int bigIntegerBytesInternalScratch() { - - if (!isSet()) { - throw new RuntimeException("no value set"); - } - - if (internalScratchLongs == null) { - internalScratchLongs = new long[FastHiveDecimal.FAST_SCRATCH_LONGS_LEN]; - internalScratchBuffer = new byte[FastHiveDecimal.FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; - } - - int byteLength = - fastBigIntegerBytes( - internalScratchLongs, internalScratchBuffer); - if (byteLength == 0) { - throw new RuntimeException("Couldn't convert decimal to binary"); - } - return byteLength; - } - - /** - * Returns the scratch array containing the result after a call to bigIntegerBytesInternalScratch. - * - */ - @HiveDecimalWritableVersionV2 - public byte[] bigIntegerBytesInternalScratchBuffer() { - return internalScratchBuffer; - } - - /** - * Allocate scratchLongs with HiveDecimal.SCRATCH_LONGS_LEN longs. - * And, allocate scratch scratchBuffer with HiveDecimal.SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes. - * - */ - @HiveDecimalWritableVersionV2 - public byte[] bigIntegerBytesCopy( - long[] scratchLongs, byte[] scratchBuffer) { - - if (!isSet()) { - throw new RuntimeException("no value set"); - } - - int byteLength = - fastBigIntegerBytes( - scratchLongs, scratchBuffer); - if (byteLength == 0) { - throw new RuntimeException("Couldn't convert decimal to binary"); - } - return Arrays.copyOf(scratchBuffer, byteLength); - } - - @HiveDecimalWritableVersionV2 - public int bigIntegerBytes( - long[] scratchLongs, byte[] scratchBuffer) { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - int byteLength = - fastBigIntegerBytes( - scratchLongs, scratchBuffer); - return byteLength; - } - - @HiveDecimalWritableVersionV2 - public int signum() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastSignum(); - } - - @HiveDecimalWritableVersionV2 - public int precision() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastSqlPrecision(); - } - - @HiveDecimalWritableVersionV2 - public int rawPrecision() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastRawPrecision(); - } - - @HiveDecimalWritableVersionV2 - public int scale() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastScale(); - } - - @HiveDecimalWritableVersionV2 - public boolean isByte() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastIsByte(); - } - - @HiveDecimalWritableVersionV2 - public byte byteValue() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastByteValueClip(); - } - - @HiveDecimalWritableVersionV2 - public boolean isShort() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastIsShort(); - } - - @HiveDecimalWritableVersionV2 - public short shortValue() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastShortValueClip(); - } - - @HiveDecimalWritableVersionV2 - public boolean isInt() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastIsInt(); - } - - @HiveDecimalWritableVersionV2 - public int intValue() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastIntValueClip(); - } - - @HiveDecimalWritableVersionV2 - public boolean isLong() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastIsLong(); - } - - @HiveDecimalWritableVersionV2 - public long longValue() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastLongValueClip(); - } - - @HiveDecimalWritableVersionV2 - public float floatValue() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastFloatValue(); - } - - @HiveDecimalWritableVersionV2 - public double doubleValue() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastDoubleValue(); - } - - //----------------------------------------------------------------------------------------------- - // Mutate operations. - //----------------------------------------------------------------------------------------------- - - @HiveDecimalWritableVersionV2 - public void mutateAbs() { - if (!isSet) { - return; - } - fastAbs(); - } - - @HiveDecimalWritableVersionV2 - public void mutateNegate() { - if (!isSet) { - return; - } - fastNegate(); - } - - @HiveDecimalWritableVersionV2 - public void mutateAdd(HiveDecimalWritable decWritable) { - if (!isSet || !decWritable.isSet) { - isSet = false; - return; - } - isSet = - fastAdd(decWritable, this); - } - - @HiveDecimalWritableVersionV2 - public void mutateAdd(HiveDecimal dec) { - if (dec == null) { - - // Can't add NULL. - isSet = false; - return; - } - if (!isSet) { - return; - } - isSet = - fastAdd(dec, this); - } - - @HiveDecimalWritableVersionV2 - public void mutateSubtract(HiveDecimalWritable decWritable) { - if (!isSet || !decWritable.isSet) { - isSet = false; - return; - } - isSet = - fastSubtract(decWritable, this); - } - - @HiveDecimalWritableVersionV2 - public void mutateSubtract(HiveDecimal dec) { - if (dec == null) { - - // Can't subtract NULL. - isSet = false; - return; - } - if (!isSet) { - return; - } - isSet = - fastSubtract(dec, this); - } - - @HiveDecimalWritableVersionV2 - public void mutateMultiply(HiveDecimalWritable decWritable) { - if (!isSet || !decWritable.isSet) { - isSet = false; - return; - } - isSet = - fastMultiply(decWritable, this); - } - - @HiveDecimalWritableVersionV2 - public void mutateMultiply(HiveDecimal dec) { - if (dec == null) { - - // Can't multiply NULL. - isSet = false; - return; - } - if (!isSet) { - return; - } - isSet = - fastMultiply(dec, this); - } - - @HiveDecimalWritableVersionV2 - public void mutateDivide(HiveDecimalWritable decWritable) { - if (!isSet || !decWritable.isSet) { - isSet = false; - return; - } - isSet = - fastDivide(decWritable, this); - } - - @HiveDecimalWritableVersionV2 - public void mutateDivide(HiveDecimal dec) { - if (dec == null) { - - // Can't divide NULL. - isSet = false; - return; - } - if (!isSet) { - return; - } - isSet = - fastDivide(dec, this); - - } - - @HiveDecimalWritableVersionV2 - public void mutateRemainder(HiveDecimalWritable decWritable) { - if (!isSet || !decWritable.isSet) { - isSet = false; - return; - } - isSet = - fastRemainder(decWritable, this); - } - - @HiveDecimalWritableVersionV2 - public void mutateRemainder(HiveDecimal dec) { - if (dec == null) { - - // Can't do remainder on NULL. - isSet = false; - return; - } - if (!isSet) { - return; - } - isSet = - fastRemainder(dec, this); - } - - @HiveDecimalWritableVersionV2 - public void mutateScaleByPowerOfTen(int power) { - if (!isSet) { - return; - } - isSet = fastScaleByPowerOfTen(power, this); - } - - @HiveDecimalWritableVersionV2 - public void mutateFractionPortion() { - if (!isSet) { - return; - } - fastFractionPortion(); - } - - @HiveDecimalWritableVersionV2 - public void mutateIntegerPortion() { - if (!isSet) { - return; - } - fastIntegerPortion(); - } - - //----------------------------------------------------------------------------------------------- - // Standard overrides methods. - //----------------------------------------------------------------------------------------------- - - @HiveDecimalWritableVersionV1 - @Override - public int compareTo(HiveDecimalWritable writable) { - if (!isSet() || writable == null || !writable.isSet()) { - throw new RuntimeException("Invalid comparision operand(s)"); - } - return fastCompareTo(writable); - } - - @HiveDecimalWritableVersionV2 - public int compareTo(HiveDecimal dec) { - if (!isSet() || dec == null) { - throw new RuntimeException("Invalid comparision operand(s)"); - } - return fastCompareTo(dec); - } - - @HiveDecimalWritableVersionV2 - public static int compareTo(HiveDecimal dec, HiveDecimalWritable writable) { - if (dec == null || !writable.isSet()) { - throw new RuntimeException("Invalid comparision operand(s)"); - } - return FastHiveDecimal.fastCompareTo(dec, writable); - } - - @HiveDecimalWritableVersionV2 - public int toBytes(byte[] scratchBuffer) { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastToBytes(scratchBuffer); - } - - @HiveDecimalWritableVersionV1 - @Override - public String toString() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastToString(); - } - - @HiveDecimalWritableVersionV2 - public String toString( - byte[] scratchBuffer) { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - if (fastSerializationScale() != -1) { - - // Use the serialization scale and format the string with trailing zeroes (or - // round the decimal) if necessary. - return - fastToFormatString( - fastSerializationScale(), - scratchBuffer); - } else { - return - fastToString(scratchBuffer); - } - } - - @HiveDecimalWritableVersionV2 - public String toFormatString( - int formatScale) { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return - fastToFormatString( - formatScale); - } - - @HiveDecimalWritableVersionV2 - public int toFormatBytes( - int formatScale, - byte[] scratchBuffer) { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return - fastToFormatBytes( - formatScale, - scratchBuffer); - } - - @HiveDecimalWritableVersionV2 - public int toDigitsOnlyBytes( - byte[] scratchBuffer) { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return - fastToDigitsOnlyBytes( - scratchBuffer); - } - - @HiveDecimalWritableVersionV1 - @Override - public boolean equals(Object other) { - if (!isSet) { - return false; - } - if (this == other) { - return true; - } - if (other == null || getClass() != other.getClass()) { - return false; - } - HiveDecimalWritable otherHiveDecWritable = (HiveDecimalWritable) other; - if (!otherHiveDecWritable.isSet()) { - return false; - } - return fastEquals((FastHiveDecimal) otherHiveDecWritable); - - } - - @HiveDecimalWritableVersionV2 - public int newFasterHashCode() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastNewFasterHashCode(); - } - - @HiveDecimalWritableVersionV1 - @Override - public int hashCode() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastHashCode(); - } - - private static final byte[] EMPTY_ARRAY = new byte[0]; - - @HiveDecimalWritableVersionV1 - public byte[] getInternalStorage() { - if (!isSet()) { - // don't break old callers that are trying to reuse storages - return EMPTY_ARRAY; - } - - if (internalScratchLongs == null) { - internalScratchLongs = new long[FastHiveDecimal.FAST_SCRATCH_LONGS_LEN]; - internalScratchBuffer = new byte[FastHiveDecimal.FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; - } - - return bigIntegerBytesCopy( - internalScratchLongs, internalScratchBuffer); - } - - @HiveDecimalWritableVersionV1 - public int getScale() { - if (!isSet()) { - throw new RuntimeException("no value set"); - } - return fastScale(); - } - - @HiveDecimalWritableVersionV2 - public void mutateSetScale(int roundingPoint, int roundingMode) { - if (!isSet) { - return; - } - isSet = fastRound(roundingPoint, roundingMode, this); - if (!isSet) { - fastReset(); - } - } - - @HiveDecimalWritableVersionV2 - public boolean mutateEnforcePrecisionScale(int precision, int scale) { - if (!isSet) { - return false; - } - isSet = fastEnforcePrecisionScale(precision, scale); - if (!isSet) { - fastReset(); - } - return isSet; - } - - @HiveDecimalWritableVersionV1 - public static HiveDecimalWritable enforcePrecisionScale(HiveDecimalWritable writable, int precision, int scale) { - if (!writable.isSet) { - return null; - } - HiveDecimalWritable result = new HiveDecimalWritable(writable); - result.mutateEnforcePrecisionScale(precision, scale); - if (!result.isSet()) { - return null; - } - return result; - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableV1.java b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableV1.java deleted file mode 100644 index ec976e9022fdd07867159afbf115ae9b24ca1c35..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableV1.java +++ /dev/null @@ -1,191 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.io; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.math.BigInteger; - -import org.apache.hadoop.hive.common.type.HiveDecimalV1; - -import org.apache.hadoop.io.WritableComparable; -import org.apache.hadoop.io.WritableUtils; - -public class HiveDecimalWritableV1 implements WritableComparable { - - private byte[] internalStorage = new byte[0]; - private int scale; - - @HiveDecimalWritableVersionV1 - public HiveDecimalWritableV1() { - } - - @HiveDecimalWritableVersionV1 - public HiveDecimalWritableV1(String value) { - set(HiveDecimalV1.create(value)); - } - - @HiveDecimalWritableVersionV1 - public HiveDecimalWritableV1(byte[] bytes, int scale) { - set(bytes, scale); - } - - @HiveDecimalWritableVersionV1 - public HiveDecimalWritableV1(HiveDecimalWritableV1 writable) { - set(writable.getHiveDecimal()); - } - - @HiveDecimalWritableVersionV1 - public HiveDecimalWritableV1(HiveDecimalV1 value) { - set(value); - } - - @HiveDecimalWritableVersionV1 - public HiveDecimalWritableV1(long value) { - set((HiveDecimalV1.create(value))); - } - - @HiveDecimalWritableVersionV1 - public void set(HiveDecimalV1 value) { - set(value.unscaledValue().toByteArray(), value.scale()); - } - - @HiveDecimalWritableVersionV1 - public void set(HiveDecimalV1 value, int maxPrecision, int maxScale) { - set(HiveDecimalV1.enforcePrecisionScale(value, maxPrecision, maxScale)); - } - - @HiveDecimalWritableVersionV1 - public void set(HiveDecimalWritableV1 writable) { - set(writable.getHiveDecimal()); - } - - @HiveDecimalWritableVersionV1 - public void set(byte[] bytes, int scale) { - this.internalStorage = bytes; - this.scale = scale; - } - - @HiveDecimalWritableVersionV1 - public HiveDecimalV1 getHiveDecimal() { - return HiveDecimalV1.create(new BigInteger(internalStorage), scale); - } - - /** - * Get a OldHiveDecimal instance from the writable and constraint it with maximum precision/scale. - * - * @param maxPrecision maximum precision - * @param maxScale maximum scale - * @return OldHiveDecimal instance - */ - @HiveDecimalWritableVersionV1 - public HiveDecimalV1 getHiveDecimal(int maxPrecision, int maxScale) { - return HiveDecimalV1.enforcePrecisionScale(HiveDecimalV1. - create(new BigInteger(internalStorage), scale), - maxPrecision, maxScale); - } - - @HiveDecimalWritableVersionV1 - @Override - public void readFields(DataInput in) throws IOException { - scale = WritableUtils.readVInt(in); - int byteArrayLen = WritableUtils.readVInt(in); - if (internalStorage.length != byteArrayLen) { - internalStorage = new byte[byteArrayLen]; - } - in.readFully(internalStorage); - } - - @HiveDecimalWritableVersionV1 - @Override - public void write(DataOutput out) throws IOException { - WritableUtils.writeVInt(out, scale); - WritableUtils.writeVInt(out, internalStorage.length); - out.write(internalStorage); - } - - @HiveDecimalWritableVersionV1 - @Override - public int compareTo(HiveDecimalWritableV1 that) { - return getHiveDecimal().compareTo(that.getHiveDecimal()); - } - - @HiveDecimalWritableVersionV1 - @Override - public String toString() { - return getHiveDecimal().toString(); - } - - @HiveDecimalWritableVersionV1 - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (other == null || getClass() != other.getClass()) { - return false; - } - HiveDecimalWritableV1 bdw = (HiveDecimalWritableV1) other; - - // 'equals' and 'compareTo' are not compatible with HiveDecimals. We want - // compareTo which returns true iff the numbers are equal (e.g.: 3.14 is - // the same as 3.140). 'Equals' returns true iff equal and the same scale - // is set in the decimals (e.g.: 3.14 is not the same as 3.140) - return getHiveDecimal().compareTo(bdw.getHiveDecimal()) == 0; - } - - @HiveDecimalWritableVersionV1 - @Override - public int hashCode() { - return getHiveDecimal().hashCode(); - } - - /* (non-Javadoc) - * In order to update a Decimal128 fast (w/o allocation) we need to expose access to the - * internal storage bytes and scale. - * @return - */ - @HiveDecimalWritableVersionV1 - public byte[] getInternalStorage() { - return internalStorage; - } - - /* (non-Javadoc) - * In order to update a Decimal128 fast (w/o allocation) we need to expose access to the - * internal storage bytes and scale. - */ - @HiveDecimalWritableVersionV1 - public int getScale() { - return scale; - } - - @HiveDecimalWritableVersionV1 - public static - HiveDecimalWritableV1 enforcePrecisionScale(HiveDecimalWritableV1 writable, - int precision, int scale) { - if (writable == null) { - return null; - } - - HiveDecimalV1 dec = - HiveDecimalV1.enforcePrecisionScale(writable.getHiveDecimal(), precision, - scale); - return dec == null ? null : new HiveDecimalWritableV1(dec); - } -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableVersionV1.java b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableVersionV1.java deleted file mode 100644 index 1aaa6f074dfc4c704fe889bf76f486f916960422..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableVersionV1.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.io; - -import java.lang.annotation.Documented; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -/** - * Marks methods including static methods and fields as being part of version 1 HiveDecimalWritable. - * - */ -@Documented -@Retention(RetentionPolicy.RUNTIME) -public @interface HiveDecimalWritableVersionV1 { - -} diff --git a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableVersionV2.java b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableVersionV2.java deleted file mode 100644 index 3c2b6f936033fc9acd9ab499c7dc50742e3d023e..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritableVersionV2.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.io; - -import java.lang.annotation.Documented; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -/** - * Marks methods including static methods and fields as being part of version 2 HiveDecimalWritable. - * - */ -@Documented -@Retention(RetentionPolicy.RUNTIME) -public @interface HiveDecimalWritableVersionV2 { - -} diff --git a/storage-api/src/java/org/apache/hive/common/util/BloomFilter.java b/storage-api/src/java/org/apache/hive/common/util/BloomFilter.java deleted file mode 100644 index 706b83409b2e8920af5ff6a6cc5d8f61398ff745..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hive/common/util/BloomFilter.java +++ /dev/null @@ -1,395 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.common.util; - -import java.io.*; -import java.util.Arrays; - -/** - * BloomFilter is a probabilistic data structure for set membership check. BloomFilters are - * highly space efficient when compared to using a HashSet. Because of the probabilistic nature of - * bloom filter false positive (element not present in bloom filter but test() says true) are - * possible but false negatives are not possible (if element is present then test() will never - * say false). The false positive probability is configurable (default: 5%) depending on which - * storage requirement may increase or decrease. Lower the false positive probability greater - * is the space requirement. - * Bloom filters are sensitive to number of elements that will be inserted in the bloom filter. - * During the creation of bloom filter expected number of entries must be specified. If the number - * of insertions exceed the specified initial number of entries then false positive probability will - * increase accordingly. - * - * Internally, this implementation of bloom filter uses Murmur3 fast non-cryptographic hash - * algorithm. Although Murmur2 is slightly faster than Murmur3 in Java, it suffers from hash - * collisions for specific sequence of repeating bytes. Check the following link for more info - * https://code.google.com/p/smhasher/wiki/MurmurHash2Flaw - */ -public class BloomFilter { - public static final double DEFAULT_FPP = 0.05; - protected BitSet bitSet; - protected int numBits; - protected int numHashFunctions; - - public BloomFilter() { - } - - public BloomFilter(long expectedEntries) { - this(expectedEntries, DEFAULT_FPP); - } - - static void checkArgument(boolean expression, String message) { - if (!expression) { - throw new IllegalArgumentException(message); - } - } - - public BloomFilter(long expectedEntries, double fpp) { - checkArgument(expectedEntries > 0, "expectedEntries should be > 0"); - checkArgument(fpp > 0.0 && fpp < 1.0, "False positive probability should be > 0.0 & < 1.0"); - int nb = optimalNumOfBits(expectedEntries, fpp); - // make 'm' multiple of 64 - this.numBits = nb + (Long.SIZE - (nb % Long.SIZE)); - this.numHashFunctions = optimalNumOfHashFunctions(expectedEntries, numBits); - this.bitSet = new BitSet(numBits); - } - - /** - * A constructor to support rebuilding the BloomFilter from a serialized representation. - * @param bits - bits are used as such for bitset and are NOT copied, any changes to bits will affect bloom filter - * @param numFuncs - number of hash functions - */ - public BloomFilter(long[] bits, int numFuncs) { - super(); - // input long[] is set as such without copying, so any modification to the source will affect bloom filter - this.bitSet = new BitSet(bits); - this.numBits = bits.length * Long.SIZE; - this.numHashFunctions = numFuncs; - } - - static int optimalNumOfHashFunctions(long n, long m) { - return Math.max(1, (int) Math.round((double) m / n * Math.log(2))); - } - - static int optimalNumOfBits(long n, double p) { - return (int) (-n * Math.log(p) / (Math.log(2) * Math.log(2))); - } - - public void add(byte[] val) { - if (val == null) { - addBytes(val, -1, -1); - } else { - addBytes(val, 0, val.length); - } - } - - public void addBytes(byte[] val, int offset, int length) { - // We use the trick mentioned in "Less Hashing, Same Performance: Building a Better Bloom Filter" - // by Kirsch et.al. From abstract 'only two hash functions are necessary to effectively - // implement a Bloom filter without any loss in the asymptotic false positive probability' - - // Lets split up 64-bit hashcode into two 32-bit hash codes and employ the technique mentioned - // in the above paper - long hash64 = val == null ? Murmur3.NULL_HASHCODE : - Murmur3.hash64(val, offset, length); - addHash(hash64); - } - - private void addHash(long hash64) { - int hash1 = (int) hash64; - int hash2 = (int) (hash64 >>> 32); - - for (int i = 1; i <= numHashFunctions; i++) { - int combinedHash = hash1 + ((i + 1) * hash2); - // hashcode should be positive, flip all the bits if it's negative - if (combinedHash < 0) { - combinedHash = ~combinedHash; - } - int pos = combinedHash % numBits; - bitSet.set(pos); - } - } - - public void addString(String val) { - if (val == null) { - add(null); - } else { - add(val.getBytes()); - } - } - - public void addLong(long val) { - addHash(getLongHash(val)); - } - - public void addDouble(double val) { - addLong(Double.doubleToLongBits(val)); - } - - public boolean test(byte[] val) { - if (val == null) { - return testBytes(val, -1, -1); - } - return testBytes(val, 0, val.length); - } - - public boolean testBytes(byte[] val, int offset, int length) { - long hash64 = val == null ? Murmur3.NULL_HASHCODE : - Murmur3.hash64(val, offset, length); - return testHash(hash64); - } - - private boolean testHash(long hash64) { - int hash1 = (int) hash64; - int hash2 = (int) (hash64 >>> 32); - - for (int i = 1; i <= numHashFunctions; i++) { - int combinedHash = hash1 + ((i + 1) * hash2); - // hashcode should be positive, flip all the bits if it's negative - if (combinedHash < 0) { - combinedHash = ~combinedHash; - } - int pos = combinedHash % numBits; - if (!bitSet.get(pos)) { - return false; - } - } - return true; - } - - public boolean testString(String val) { - if (val == null) { - return test(null); - } else { - return test(val.getBytes()); - } - } - - public boolean testLong(long val) { - return testHash(getLongHash(val)); - } - - // Thomas Wang's integer hash function - // http://web.archive.org/web/20071223173210/http://www.concentric.net/~Ttwang/tech/inthash.htm - private long getLongHash(long key) { - key = (~key) + (key << 21); // key = (key << 21) - key - 1; - key = key ^ (key >> 24); - key = (key + (key << 3)) + (key << 8); // key * 265 - key = key ^ (key >> 14); - key = (key + (key << 2)) + (key << 4); // key * 21 - key = key ^ (key >> 28); - key = key + (key << 31); - return key; - } - - public boolean testDouble(double val) { - return testLong(Double.doubleToLongBits(val)); - } - - public long sizeInBytes() { - return getBitSize() / 8; - } - - public int getBitSize() { - return bitSet.getData().length * Long.SIZE; - } - - public int getNumHashFunctions() { - return numHashFunctions; - } - - public long[] getBitSet() { - return bitSet.getData(); - } - - @Override - public String toString() { - return "m: " + numBits + " k: " + numHashFunctions; - } - - /** - * Merge the specified bloom filter with current bloom filter. - * - * @param that - bloom filter to merge - */ - public void merge(BloomFilter that) { - if (this != that && this.numBits == that.numBits && this.numHashFunctions == that.numHashFunctions) { - this.bitSet.putAll(that.bitSet); - } else { - throw new IllegalArgumentException("BloomFilters are not compatible for merging." + - " this - " + this.toString() + " that - " + that.toString()); - } - } - - public void reset() { - this.bitSet.clear(); - } - - /** - * Serialize a bloom filter - * @param out output stream to write to - * @param bloomFilter BloomFilter that needs to be seralized - */ - public static void serialize(OutputStream out, BloomFilter bloomFilter) throws IOException { - /** - * Serialized BloomFilter format: - * 1 byte for the number of hash functions. - * 1 big endian int(That is how OutputStream works) for the number of longs in the bitset - * big endian longs in the BloomFilter bitset - */ - DataOutputStream dataOutputStream = new DataOutputStream(out); - dataOutputStream.writeByte(bloomFilter.numHashFunctions); - dataOutputStream.writeInt(bloomFilter.getBitSet().length); - for (long value : bloomFilter.getBitSet()) { - dataOutputStream.writeLong(value); - } - } - - /** - * Deserialize a bloom filter - * Read a byte stream, which was written by {@linkplain #serialize(OutputStream, BloomFilter)} - * into a {@code BloomFilter} - * @param in input bytestream - * @return deserialized BloomFilter - */ - public static BloomFilter deserialize(InputStream in) throws IOException { - if (in == null) { - throw new IOException("Input stream is null"); - } - - try { - DataInputStream dataInputStream = new DataInputStream(in); - int numHashFunc = dataInputStream.readByte(); - int numLongs = dataInputStream.readInt(); - long[] data = new long[numLongs]; - for (int i = 0; i < numLongs; i++) { - data[i] = dataInputStream.readLong(); - } - return new BloomFilter(data, numHashFunc); - } catch (RuntimeException e) { - IOException io = new IOException( "Unable to deserialize BloomFilter"); - io.initCause(e); - throw io; - } - } - - // Given a byte array consisting of a serialized BloomFilter, gives the offset (from 0) - // for the start of the serialized long values that make up the bitset. - // NumHashFunctions (1 byte) + NumBits (4 bytes) - public static final int START_OF_SERIALIZED_LONGS = 5; - - /** - * Merges BloomFilter bf2 into bf1. - * Assumes 2 BloomFilters with the same size/hash functions are serialized to byte arrays - * @param bf1Bytes - * @param bf1Start - * @param bf1Length - * @param bf2Bytes - * @param bf2Start - * @param bf2Length - */ - public static void mergeBloomFilterBytes( - byte[] bf1Bytes, int bf1Start, int bf1Length, - byte[] bf2Bytes, int bf2Start, int bf2Length) { - if (bf1Length != bf2Length) { - throw new IllegalArgumentException("bf1Length " + bf1Length + " does not match bf2Length " + bf2Length); - } - - // Validation on the bitset size/3 hash functions. - for (int idx = 0; idx < START_OF_SERIALIZED_LONGS; ++idx) { - if (bf1Bytes[bf1Start + idx] != bf2Bytes[bf2Start + idx]) { - throw new IllegalArgumentException("bf1 NumHashFunctions/NumBits does not match bf2"); - } - } - - // Just bitwise-OR the bits together - size/# functions should be the same, - // rest of the data is serialized long values for the bitset which are supposed to be bitwise-ORed. - for (int idx = START_OF_SERIALIZED_LONGS; idx < bf1Length; ++idx) { - bf1Bytes[bf1Start + idx] |= bf2Bytes[bf2Start + idx]; - } - } - - /** - * Bare metal bit set implementation. For performance reasons, this implementation does not check - * for index bounds nor expand the bit set size if the specified index is greater than the size. - */ - public class BitSet { - private final long[] data; - - public BitSet(long bits) { - this(new long[(int) Math.ceil((double) bits / (double) Long.SIZE)]); - } - - /** - * Deserialize long array as bit set. - * - * @param data - bit array - */ - public BitSet(long[] data) { - assert data.length > 0 : "data length is zero!"; - this.data = data; - } - - /** - * Sets the bit at specified index. - * - * @param index - position - */ - public void set(int index) { - data[index >>> 6] |= (1L << index); - } - - /** - * Returns true if the bit is set in the specified index. - * - * @param index - position - * @return - value at the bit position - */ - public boolean get(int index) { - return (data[index >>> 6] & (1L << index)) != 0; - } - - /** - * Number of bits - */ - public long bitSize() { - return (long) data.length * Long.SIZE; - } - - public long[] getData() { - return data; - } - - /** - * Combines the two BitArrays using bitwise OR. - */ - public void putAll(BitSet array) { - assert data.length == array.data.length : - "BitArrays must be of equal length (" + data.length + "!= " + array.data.length + ")"; - for (int i = 0; i < data.length; i++) { - data[i] |= array.data[i]; - } - } - - /** - * Clear the bit set. - */ - public void clear() { - Arrays.fill(data, 0); - } - } -} diff --git a/storage-api/src/java/org/apache/hive/common/util/BloomKFilter.java b/storage-api/src/java/org/apache/hive/common/util/BloomKFilter.java deleted file mode 100644 index 45326ab009763c0560b474baba2d39ad77e13416..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hive/common/util/BloomKFilter.java +++ /dev/null @@ -1,472 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.common.util; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Arrays; - -/** - * BloomKFilter is variation of {@link BloomFilter}. Unlike BloomFilter, BloomKFilter will spread - * 'k' hash bits within same cache line for better L1 cache performance. The way it works is, - * First hash code is computed from key which is used to locate the block offset (n-longs in bitset constitute a block) - * Subsequent 'k' hash codes are used to spread hash bits within the block. By default block size is chosen as 8, - * which is to match cache line size (8 longs = 64 bytes = cache line size). - * Refer {@link BloomKFilter#addBytes(byte[])} for more info. - * - * This implementation has much lesser L1 data cache misses than {@link BloomFilter}. - */ -public class BloomKFilter { - private byte[] BYTE_ARRAY_4 = new byte[4]; - private byte[] BYTE_ARRAY_8 = new byte[8]; - public static final float DEFAULT_FPP = 0.05f; - private static final int DEFAULT_BLOCK_SIZE = 8; - private static final int DEFAULT_BLOCK_SIZE_BITS = (int) (Math.log(DEFAULT_BLOCK_SIZE) / Math.log(2)); - private static final int DEFAULT_BLOCK_OFFSET_MASK = DEFAULT_BLOCK_SIZE - 1; - private static final int DEFAULT_BIT_OFFSET_MASK = Long.SIZE - 1; - private final long[] masks = new long[DEFAULT_BLOCK_SIZE]; - private BitSet bitSet; - private final int m; - private final int k; - // spread k-1 bits to adjacent longs, default is 8 - // spreading hash bits within blockSize * longs will make bloom filter L1 cache friendly - // default block size is set to 8 as most cache line sizes are 64 bytes and also AVX512 friendly - private final int totalBlockCount; - - static void checkArgument(boolean expression, String message) { - if (!expression) { - throw new IllegalArgumentException(message); - } - } - - public BloomKFilter(long maxNumEntries) { - checkArgument(maxNumEntries > 0, "expectedEntries should be > 0"); - long numBits = optimalNumOfBits(maxNumEntries, DEFAULT_FPP); - this.k = optimalNumOfHashFunctions(maxNumEntries, numBits); - int nLongs = (int) Math.ceil((double) numBits / (double) Long.SIZE); - // additional bits to pad long array to block size - int padLongs = DEFAULT_BLOCK_SIZE - nLongs % DEFAULT_BLOCK_SIZE; - this.m = (nLongs + padLongs) * Long.SIZE; - this.bitSet = new BitSet(m); - checkArgument((bitSet.data.length % DEFAULT_BLOCK_SIZE) == 0, "bitSet has to be block aligned"); - this.totalBlockCount = bitSet.data.length / DEFAULT_BLOCK_SIZE; - } - - /** - * A constructor to support rebuilding the BloomFilter from a serialized representation. - * @param bits - * @param numFuncs - */ - public BloomKFilter(long[] bits, int numFuncs) { - super(); - bitSet = new BitSet(bits); - this.m = bits.length * Long.SIZE; - this.k = numFuncs; - checkArgument((bitSet.data.length % DEFAULT_BLOCK_SIZE) == 0, "bitSet has to be block aligned"); - this.totalBlockCount = bitSet.data.length / DEFAULT_BLOCK_SIZE; - } - static int optimalNumOfHashFunctions(long n, long m) { - return Math.max(1, (int) Math.round((double) m / n * Math.log(2))); - } - - static long optimalNumOfBits(long n, double p) { - return (long) (-n * Math.log(p) / (Math.log(2) * Math.log(2))); - } - - public void add(byte[] val) { - addBytes(val); - } - - public void addBytes(byte[] val, int offset, int length) { - // We use the trick mentioned in "Less Hashing, Same Performance: Building a Better Bloom Filter" - // by Kirsch et.al. From abstract 'only two hash functions are necessary to effectively - // implement a Bloom filter without any loss in the asymptotic false positive probability' - - // Lets split up 64-bit hashcode into two 32-bit hash codes and employ the technique mentioned - // in the above paper - long hash64 = val == null ? Murmur3.NULL_HASHCODE : - Murmur3.hash64(val, offset, length); - addHash(hash64); - } - - public void addBytes(byte[] val) { - addBytes(val, 0, val.length); - } - - private void addHash(long hash64) { - final int hash1 = (int) hash64; - final int hash2 = (int) (hash64 >>> 32); - - int firstHash = hash1 + hash2; - // hashcode should be positive, flip all the bits if it's negative - if (firstHash < 0) { - firstHash = ~firstHash; - } - - // first hash is used to locate start of the block (blockBaseOffset) - // subsequent K hashes are used to generate K bits within a block of words - final int blockIdx = firstHash % totalBlockCount; - final int blockBaseOffset = blockIdx << DEFAULT_BLOCK_SIZE_BITS; - for (int i = 1; i <= k; i++) { - int combinedHash = hash1 + ((i + 1) * hash2); - // hashcode should be positive, flip all the bits if it's negative - if (combinedHash < 0) { - combinedHash = ~combinedHash; - } - // LSB 3 bits is used to locate offset within the block - final int absOffset = blockBaseOffset + (combinedHash & DEFAULT_BLOCK_OFFSET_MASK); - // Next 6 bits are used to locate offset within a long/word - final int bitPos = (combinedHash >>> DEFAULT_BLOCK_SIZE_BITS) & DEFAULT_BIT_OFFSET_MASK; - bitSet.data[absOffset] |= (1L << bitPos); - } - } - - public void addString(String val) { - addBytes(val.getBytes()); - } - - public void addByte(byte val) { - addBytes(new byte[]{val}); - } - - public void addInt(int val) { - // puts int in little endian order - addBytes(intToByteArrayLE(val)); - } - - - public void addLong(long val) { - // puts long in little endian order - addBytes(longToByteArrayLE(val)); - } - - public void addFloat(float val) { - addInt(Float.floatToIntBits(val)); - } - - public void addDouble(double val) { - addLong(Double.doubleToLongBits(val)); - } - - public boolean test(byte[] val) { - return testBytes(val); - } - - public boolean testBytes(byte[] val) { - return testBytes(val, 0, val.length); - } - - public boolean testBytes(byte[] val, int offset, int length) { - long hash64 = val == null ? Murmur3.NULL_HASHCODE : - Murmur3.hash64(val, offset, length); - return testHash(hash64); - } - - private boolean testHash(long hash64) { - final int hash1 = (int) hash64; - final int hash2 = (int) (hash64 >>> 32); - - int firstHash = hash1 + hash2; - // hashcode should be positive, flip all the bits if it's negative - if (firstHash < 0) { - firstHash = ~firstHash; - } - - // first hash is used to locate start of the block (blockBaseOffset) - // subsequent K hashes are used to generate K bits within a block of words - // To avoid branches during probe, a separate masks array is used for each longs/words within a block. - // data array and masks array are then traversed together and checked for corresponding set bits. - final int blockIdx = firstHash % totalBlockCount; - final int blockBaseOffset = blockIdx << DEFAULT_BLOCK_SIZE_BITS; - - // iterate and update masks array - for (int i = 1; i <= k; i++) { - int combinedHash = hash1 + ((i + 1) * hash2); - // hashcode should be positive, flip all the bits if it's negative - if (combinedHash < 0) { - combinedHash = ~combinedHash; - } - // LSB 3 bits is used to locate offset within the block - final int wordOffset = combinedHash & DEFAULT_BLOCK_OFFSET_MASK; - // Next 6 bits are used to locate offset within a long/word - final int bitPos = (combinedHash >>> DEFAULT_BLOCK_SIZE_BITS) & DEFAULT_BIT_OFFSET_MASK; - masks[wordOffset] |= (1L << bitPos); - } - - // traverse data and masks array together, check for set bits - long expected = 0; - for (int i = 0; i < DEFAULT_BLOCK_SIZE; i++) { - final long mask = masks[i]; - expected |= (bitSet.data[blockBaseOffset + i] & mask) ^ mask; - } - - // clear the mask for array reuse (this is to avoid masks array allocation in inner loop) - Arrays.fill(masks, 0); - - // if all bits are set, expected should be 0 - return expected == 0; - } - - public boolean testString(String val) { - return testBytes(val.getBytes()); - } - - public boolean testByte(byte val) { - return testBytes(new byte[]{val}); - } - - public boolean testInt(int val) { - return testBytes(intToByteArrayLE(val)); - } - - public boolean testLong(long val) { - return testBytes(longToByteArrayLE(val)); - } - - public boolean testFloat(float val) { - return testInt(Float.floatToIntBits(val)); - } - - public boolean testDouble(double val) { - return testLong(Double.doubleToLongBits(val)); - } - - private byte[] intToByteArrayLE(int val) { - BYTE_ARRAY_4[0] = (byte) (val >> 0); - BYTE_ARRAY_4[1] = (byte) (val >> 8); - BYTE_ARRAY_4[2] = (byte) (val >> 16); - BYTE_ARRAY_4[3] = (byte) (val >> 24); - return BYTE_ARRAY_4; - } - - private byte[] longToByteArrayLE(long val) { - BYTE_ARRAY_8[0] = (byte) (val >> 0); - BYTE_ARRAY_8[1] = (byte) (val >> 8); - BYTE_ARRAY_8[2] = (byte) (val >> 16); - BYTE_ARRAY_8[3] = (byte) (val >> 24); - BYTE_ARRAY_8[4] = (byte) (val >> 32); - BYTE_ARRAY_8[5] = (byte) (val >> 40); - BYTE_ARRAY_8[6] = (byte) (val >> 48); - BYTE_ARRAY_8[7] = (byte) (val >> 56); - return BYTE_ARRAY_8; - } - - public long sizeInBytes() { - return getBitSize() / 8; - } - - public int getBitSize() { - return bitSet.getData().length * Long.SIZE; - } - - public int getNumHashFunctions() { - return k; - } - - public int getNumBits() { - return m; - } - - public long[] getBitSet() { - return bitSet.getData(); - } - - @Override - public String toString() { - return "m: " + m + " k: " + k; - } - - /** - * Merge the specified bloom filter with current bloom filter. - * - * @param that - bloom filter to merge - */ - public void merge(BloomKFilter that) { - if (this != that && this.m == that.m && this.k == that.k) { - this.bitSet.putAll(that.bitSet); - } else { - throw new IllegalArgumentException("BloomKFilters are not compatible for merging." + - " this - " + this.toString() + " that - " + that.toString()); - } - } - - public void reset() { - this.bitSet.clear(); - } - - /** - * Serialize a bloom filter - * - * @param out output stream to write to - * @param bloomFilter BloomKFilter that needs to be seralized - */ - public static void serialize(OutputStream out, BloomKFilter bloomFilter) throws IOException { - /** - * Serialized BloomKFilter format: - * 1 byte for the number of hash functions. - * 1 big endian int(That is how OutputStream works) for the number of longs in the bitset - * big endina longs in the BloomKFilter bitset - */ - DataOutputStream dataOutputStream = new DataOutputStream(out); - dataOutputStream.writeByte(bloomFilter.k); - dataOutputStream.writeInt(bloomFilter.getBitSet().length); - for (long value : bloomFilter.getBitSet()) { - dataOutputStream.writeLong(value); - } - } - - /** - * Deserialize a bloom filter - * Read a byte stream, which was written by {@linkplain #serialize(OutputStream, BloomKFilter)} - * into a {@code BloomKFilter} - * - * @param in input bytestream - * @return deserialized BloomKFilter - */ - public static BloomKFilter deserialize(InputStream in) throws IOException { - if (in == null) { - throw new IOException("Input stream is null"); - } - - try { - DataInputStream dataInputStream = new DataInputStream(in); - int numHashFunc = dataInputStream.readByte(); - int bitsetArrayLen = dataInputStream.readInt(); - long[] data = new long[bitsetArrayLen]; - for (int i = 0; i < bitsetArrayLen; i++) { - data[i] = dataInputStream.readLong(); - } - return new BloomKFilter(data, numHashFunc); - } catch (RuntimeException e) { - IOException io = new IOException("Unable to deserialize BloomKFilter"); - io.initCause(e); - throw io; - } - } - - // Given a byte array consisting of a serialized BloomKFilter, gives the offset (from 0) - // for the start of the serialized long values that make up the bitset. - // NumHashFunctions (1 byte) + bitset array length (4 bytes) - public static final int START_OF_SERIALIZED_LONGS = 5; - - /** - * Merges BloomKFilter bf2 into bf1. - * Assumes 2 BloomKFilters with the same size/hash functions are serialized to byte arrays - * - * @param bf1Bytes - * @param bf1Start - * @param bf1Length - * @param bf2Bytes - * @param bf2Start - * @param bf2Length - */ - public static void mergeBloomFilterBytes( - byte[] bf1Bytes, int bf1Start, int bf1Length, - byte[] bf2Bytes, int bf2Start, int bf2Length) { - if (bf1Length != bf2Length) { - throw new IllegalArgumentException("bf1Length " + bf1Length + " does not match bf2Length " + bf2Length); - } - - // Validation on the bitset size/3 hash functions. - for (int idx = 0; idx < START_OF_SERIALIZED_LONGS; ++idx) { - if (bf1Bytes[bf1Start + idx] != bf2Bytes[bf2Start + idx]) { - throw new IllegalArgumentException("bf1 NumHashFunctions/NumBits does not match bf2"); - } - } - - // Just bitwise-OR the bits together - size/# functions should be the same, - // rest of the data is serialized long values for the bitset which are supposed to be bitwise-ORed. - for (int idx = START_OF_SERIALIZED_LONGS; idx < bf1Length; ++idx) { - bf1Bytes[bf1Start + idx] |= bf2Bytes[bf2Start + idx]; - } - } - - /** - * Bare metal bit set implementation. For performance reasons, this implementation does not check - * for index bounds nor expand the bit set size if the specified index is greater than the size. - */ - public static class BitSet { - private final long[] data; - - public BitSet(long bits) { - this(new long[(int) Math.ceil((double) bits / (double) Long.SIZE)]); - } - - /** - * Deserialize long array as bit set. - * - * @param data - bit array - */ - public BitSet(long[] data) { - assert data.length > 0 : "data length is zero!"; - this.data = data; - } - - /** - * Sets the bit at specified index. - * - * @param index - position - */ - public void set(int index) { - data[index >>> 6] |= (1L << index); - } - - /** - * Returns true if the bit is set in the specified index. - * - * @param index - position - * @return - value at the bit position - */ - public boolean get(int index) { - return (data[index >>> 6] & (1L << index)) != 0; - } - - /** - * Number of bits - */ - public int bitSize() { - return data.length * Long.SIZE; - } - - public long[] getData() { - return data; - } - - /** - * Combines the two BitArrays using bitwise OR. - */ - public void putAll(BloomKFilter.BitSet array) { - assert data.length == array.data.length : - "BitArrays must be of equal length (" + data.length + "!= " + array.data.length + ")"; - for (int i = 0; i < data.length; i++) { - data[i] |= array.data[i]; - } - } - - /** - * Clear the bit set. - */ - public void clear() { - Arrays.fill(data, 0); - } - } -} diff --git a/storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java b/storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java deleted file mode 100644 index 727c1e6699479577cbcbee22d7bee13a6f299c02..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.common.util; - -import java.math.BigDecimal; -import java.text.SimpleDateFormat; - -import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; - - -/** - * DateUtils. Thread-safe class - * - */ -public class IntervalDayTimeUtils { - - private static final ThreadLocal dateFormatLocal = new ThreadLocal() { - @Override - protected SimpleDateFormat initialValue() { - return new SimpleDateFormat("yyyy-MM-dd"); - } - }; - - public static SimpleDateFormat getDateFormat() { - return dateFormatLocal.get(); - } - - public static final int NANOS_PER_SEC = 1000000000; - public static final BigDecimal MAX_INT_BD = new BigDecimal(Integer.MAX_VALUE); - public static final BigDecimal NANOS_PER_SEC_BD = new BigDecimal(NANOS_PER_SEC); - - public static int parseNumericValueWithRange(String fieldName, - String strVal, int minValue, int maxValue) throws IllegalArgumentException { - int result = 0; - if (strVal != null) { - result = Integer.parseInt(strVal); - if (result < minValue || result > maxValue) { - throw new IllegalArgumentException(String.format("%s value %d outside range [%d, %d]", - fieldName, result, minValue, maxValue)); - } - } - return result; - } - - public static long getIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime) { - return intervalDayTime.getTotalSeconds() * NANOS_PER_SEC + intervalDayTime.getNanos(); - } - - public static void setIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime, - long totalNanos) { - intervalDayTime.set(totalNanos / NANOS_PER_SEC, (int) (totalNanos % NANOS_PER_SEC)); - } - - public static long getIntervalDayTimeTotalSecondsFromTotalNanos(long totalNanos) { - return totalNanos / NANOS_PER_SEC; - } - - public static int getIntervalDayTimeNanosFromTotalNanos(long totalNanos) { - return (int) (totalNanos % NANOS_PER_SEC); - } -} diff --git a/storage-api/src/java/org/apache/hive/common/util/Murmur3.java b/storage-api/src/java/org/apache/hive/common/util/Murmur3.java deleted file mode 100644 index eb050820d67a8606e10d190cb797aedbeb219f7e..0000000000000000000000000000000000000000 --- a/storage-api/src/java/org/apache/hive/common/util/Murmur3.java +++ /dev/null @@ -1,431 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.common.util; - -/** - * Murmur3 is successor to Murmur2 fast non-crytographic hash algorithms. - * - * Murmur3 32 and 128 bit variants. - * 32-bit Java port of https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp#94 - * 128-bit Java port of https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp#255 - * - * This is a public domain code with no copyrights. - * From homepage of MurmurHash (https://code.google.com/p/smhasher/), - * "All MurmurHash versions are public domain software, and the author disclaims all copyright - * to their code." - */ -public class Murmur3 { - // from 64-bit linear congruential generator - public static final long NULL_HASHCODE = 2862933555777941757L; - - // Constants for 32 bit variant - private static final int C1_32 = 0xcc9e2d51; - private static final int C2_32 = 0x1b873593; - private static final int R1_32 = 15; - private static final int R2_32 = 13; - private static final int M_32 = 5; - private static final int N_32 = 0xe6546b64; - - // Constants for 128 bit variant - private static final long C1 = 0x87c37b91114253d5L; - private static final long C2 = 0x4cf5ad432745937fL; - private static final int R1 = 31; - private static final int R2 = 27; - private static final int R3 = 33; - private static final int M = 5; - private static final int N1 = 0x52dce729; - private static final int N2 = 0x38495ab5; - - public static final int DEFAULT_SEED = 104729; - - /** - * Murmur3 32-bit variant. - * - * @param data - input byte array - * @return - hashcode - */ - public static int hash32(byte[] data) { - return hash32(data, data.length, DEFAULT_SEED); - } - - /** - * Murmur3 32-bit variant. - * - * @param data - input byte array - * @param length - length of array - * @param seed - seed. (default 0) - * @return - hashcode - */ - public static int hash32(byte[] data, int length, int seed) { - int hash = seed; - final int nblocks = length >> 2; - - // body - for (int i = 0; i < nblocks; i++) { - int i_4 = i << 2; - int k = (data[i_4] & 0xff) - | ((data[i_4 + 1] & 0xff) << 8) - | ((data[i_4 + 2] & 0xff) << 16) - | ((data[i_4 + 3] & 0xff) << 24); - - // mix functions - k *= C1_32; - k = Integer.rotateLeft(k, R1_32); - k *= C2_32; - hash ^= k; - hash = Integer.rotateLeft(hash, R2_32) * M_32 + N_32; - } - - // tail - int idx = nblocks << 2; - int k1 = 0; - switch (length - idx) { - case 3: - k1 ^= data[idx + 2] << 16; - case 2: - k1 ^= data[idx + 1] << 8; - case 1: - k1 ^= data[idx]; - - // mix functions - k1 *= C1_32; - k1 = Integer.rotateLeft(k1, R1_32); - k1 *= C2_32; - hash ^= k1; - } - - // finalization - hash ^= length; - hash ^= (hash >>> 16); - hash *= 0x85ebca6b; - hash ^= (hash >>> 13); - hash *= 0xc2b2ae35; - hash ^= (hash >>> 16); - - return hash; - } - - /** - * Murmur3 64-bit variant. This is essentially MSB 8 bytes of Murmur3 128-bit variant. - * - * @param data - input byte array - * @return - hashcode - */ - public static long hash64(byte[] data) { - return hash64(data, 0, data.length, DEFAULT_SEED); - } - - public static long hash64(byte[] data, int offset, int length) { - return hash64(data, offset, length, DEFAULT_SEED); - } - - /** - * Murmur3 64-bit variant. This is essentially MSB 8 bytes of Murmur3 128-bit variant. - * - * @param data - input byte array - * @param length - length of array - * @param seed - seed. (default is 0) - * @return - hashcode - */ - public static long hash64(byte[] data, int offset, int length, int seed) { - long hash = seed; - final int nblocks = length >> 3; - - // body - for (int i = 0; i < nblocks; i++) { - final int i8 = i << 3; - long k = ((long) data[offset + i8] & 0xff) - | (((long) data[offset + i8 + 1] & 0xff) << 8) - | (((long) data[offset + i8 + 2] & 0xff) << 16) - | (((long) data[offset + i8 + 3] & 0xff) << 24) - | (((long) data[offset + i8 + 4] & 0xff) << 32) - | (((long) data[offset + i8 + 5] & 0xff) << 40) - | (((long) data[offset + i8 + 6] & 0xff) << 48) - | (((long) data[offset + i8 + 7] & 0xff) << 56); - - // mix functions - k *= C1; - k = Long.rotateLeft(k, R1); - k *= C2; - hash ^= k; - hash = Long.rotateLeft(hash, R2) * M + N1; - } - - // tail - long k1 = 0; - int tailStart = nblocks << 3; - switch (length - tailStart) { - case 7: - k1 ^= ((long) data[offset + tailStart + 6] & 0xff) << 48; - case 6: - k1 ^= ((long) data[offset + tailStart + 5] & 0xff) << 40; - case 5: - k1 ^= ((long) data[offset + tailStart + 4] & 0xff) << 32; - case 4: - k1 ^= ((long) data[offset + tailStart + 3] & 0xff) << 24; - case 3: - k1 ^= ((long) data[offset + tailStart + 2] & 0xff) << 16; - case 2: - k1 ^= ((long) data[offset + tailStart + 1] & 0xff) << 8; - case 1: - k1 ^= ((long) data[offset + tailStart] & 0xff); - k1 *= C1; - k1 = Long.rotateLeft(k1, R1); - k1 *= C2; - hash ^= k1; - } - - // finalization - hash ^= length; - hash = fmix64(hash); - - return hash; - } - - /** - * Murmur3 128-bit variant. - * - * @param data - input byte array - * @return - hashcode (2 longs) - */ - public static long[] hash128(byte[] data) { - return hash128(data, 0, data.length, DEFAULT_SEED); - } - - /** - * Murmur3 128-bit variant. - * - * @param data - input byte array - * @param offset - the first element of array - * @param length - length of array - * @param seed - seed. (default is 0) - * @return - hashcode (2 longs) - */ - public static long[] hash128(byte[] data, int offset, int length, int seed) { - long h1 = seed; - long h2 = seed; - final int nblocks = length >> 4; - - // body - for (int i = 0; i < nblocks; i++) { - final int i16 = i << 4; - long k1 = ((long) data[offset + i16] & 0xff) - | (((long) data[offset + i16 + 1] & 0xff) << 8) - | (((long) data[offset + i16 + 2] & 0xff) << 16) - | (((long) data[offset + i16 + 3] & 0xff) << 24) - | (((long) data[offset + i16 + 4] & 0xff) << 32) - | (((long) data[offset + i16 + 5] & 0xff) << 40) - | (((long) data[offset + i16 + 6] & 0xff) << 48) - | (((long) data[offset + i16 + 7] & 0xff) << 56); - - long k2 = ((long) data[offset + i16 + 8] & 0xff) - | (((long) data[offset + i16 + 9] & 0xff) << 8) - | (((long) data[offset + i16 + 10] & 0xff) << 16) - | (((long) data[offset + i16 + 11] & 0xff) << 24) - | (((long) data[offset + i16 + 12] & 0xff) << 32) - | (((long) data[offset + i16 + 13] & 0xff) << 40) - | (((long) data[offset + i16 + 14] & 0xff) << 48) - | (((long) data[offset + i16 + 15] & 0xff) << 56); - - // mix functions for k1 - k1 *= C1; - k1 = Long.rotateLeft(k1, R1); - k1 *= C2; - h1 ^= k1; - h1 = Long.rotateLeft(h1, R2); - h1 += h2; - h1 = h1 * M + N1; - - // mix functions for k2 - k2 *= C2; - k2 = Long.rotateLeft(k2, R3); - k2 *= C1; - h2 ^= k2; - h2 = Long.rotateLeft(h2, R1); - h2 += h1; - h2 = h2 * M + N2; - } - - // tail - long k1 = 0; - long k2 = 0; - int tailStart = nblocks << 4; - switch (length - tailStart) { - case 15: - k2 ^= (long) (data[offset + tailStart + 14] & 0xff) << 48; - case 14: - k2 ^= (long) (data[offset + tailStart + 13] & 0xff) << 40; - case 13: - k2 ^= (long) (data[offset + tailStart + 12] & 0xff) << 32; - case 12: - k2 ^= (long) (data[offset + tailStart + 11] & 0xff) << 24; - case 11: - k2 ^= (long) (data[offset + tailStart + 10] & 0xff) << 16; - case 10: - k2 ^= (long) (data[offset + tailStart + 9] & 0xff) << 8; - case 9: - k2 ^= (long) (data[offset + tailStart + 8] & 0xff); - k2 *= C2; - k2 = Long.rotateLeft(k2, R3); - k2 *= C1; - h2 ^= k2; - - case 8: - k1 ^= (long) (data[offset + tailStart + 7] & 0xff) << 56; - case 7: - k1 ^= (long) (data[offset + tailStart + 6] & 0xff) << 48; - case 6: - k1 ^= (long) (data[offset + tailStart + 5] & 0xff) << 40; - case 5: - k1 ^= (long) (data[offset + tailStart + 4] & 0xff) << 32; - case 4: - k1 ^= (long) (data[offset + tailStart + 3] & 0xff) << 24; - case 3: - k1 ^= (long) (data[offset + tailStart + 2] & 0xff) << 16; - case 2: - k1 ^= (long) (data[offset + tailStart + 1] & 0xff) << 8; - case 1: - k1 ^= (long) (data[offset + tailStart] & 0xff); - k1 *= C1; - k1 = Long.rotateLeft(k1, R1); - k1 *= C2; - h1 ^= k1; - } - - // finalization - h1 ^= length; - h2 ^= length; - - h1 += h2; - h2 += h1; - - h1 = fmix64(h1); - h2 = fmix64(h2); - - h1 += h2; - h2 += h1; - - return new long[]{h1, h2}; - } - - private static long fmix64(long h) { - h ^= (h >>> 33); - h *= 0xff51afd7ed558ccdL; - h ^= (h >>> 33); - h *= 0xc4ceb9fe1a85ec53L; - h ^= (h >>> 33); - return h; - } - - public static class IncrementalHash32 { - byte[] tail = new byte[3]; - int tailLen; - int totalLen; - int hash; - - public final void start(int hash) { - tailLen = totalLen = 0; - this.hash = hash; - } - - public final void add(byte[] data, int offset, int length) { - if (length == 0) return; - totalLen += length; - if (tailLen + length < 4) { - System.arraycopy(data, offset, tail, tailLen, length); - tailLen += length; - return; - } - int offset2 = 0; - if (tailLen > 0) { - offset2 = (4 - tailLen); - int k = -1; - switch (tailLen) { - case 1: - k = orBytes(tail[0], data[offset], data[offset + 1], data[offset + 2]); - break; - case 2: - k = orBytes(tail[0], tail[1], data[offset], data[offset + 1]); - break; - case 3: - k = orBytes(tail[0], tail[1], tail[2], data[offset]); - break; - default: throw new AssertionError(tailLen); - } - // mix functions - k *= C1_32; - k = Integer.rotateLeft(k, R1_32); - k *= C2_32; - hash ^= k; - hash = Integer.rotateLeft(hash, R2_32) * M_32 + N_32; - } - int length2 = length - offset2; - offset += offset2; - final int nblocks = length2 >> 2; - - for (int i = 0; i < nblocks; i++) { - int i_4 = (i << 2) + offset; - int k = orBytes(data[i_4], data[i_4 + 1], data[i_4 + 2], data[i_4 + 3]); - - // mix functions - k *= C1_32; - k = Integer.rotateLeft(k, R1_32); - k *= C2_32; - hash ^= k; - hash = Integer.rotateLeft(hash, R2_32) * M_32 + N_32; - } - - int consumed = (nblocks << 2); - tailLen = length2 - consumed; - if (consumed == length2) return; - System.arraycopy(data, offset + consumed, tail, 0, tailLen); - } - - public final int end() { - int k1 = 0; - switch (tailLen) { - case 3: - k1 ^= tail[2] << 16; - case 2: - k1 ^= tail[1] << 8; - case 1: - k1 ^= tail[0]; - - // mix functions - k1 *= C1_32; - k1 = Integer.rotateLeft(k1, R1_32); - k1 *= C2_32; - hash ^= k1; - } - - // finalization - hash ^= totalLen; - hash ^= (hash >>> 16); - hash *= 0x85ebca6b; - hash ^= (hash >>> 13); - hash *= 0xc2b2ae35; - hash ^= (hash >>> 16); - return hash; - } - } - - private static int orBytes(byte b1, byte b2, byte b3, byte b4) { - return (b1 & 0xff) | ((b2 & 0xff) << 8) | ((b3 & 0xff) << 16) | ((b4 & 0xff) << 24); - } -} diff --git a/storage-api/src/test/org/apache/hadoop/hive/VersionTestBase.java b/storage-api/src/test/org/apache/hadoop/hive/VersionTestBase.java deleted file mode 100644 index 489496e418a6fede80026357f49b50f2dc166e86..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/VersionTestBase.java +++ /dev/null @@ -1,322 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive; - -import java.lang.annotation.Annotation; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.TreeSet; - -import org.junit.*; - -import static org.junit.Assert.*; - -public class VersionTestBase { - - public String getParameterTypeName(Class parameterType, - Map versionedClassToNameMap) { - if (versionedClassToNameMap.containsKey(parameterType)) { - return versionedClassToNameMap.get(parameterType); - } else { - return parameterType.getSimpleName(); - } - } - - public String getMethodKey(Method method, Map versionedClassToNameMap) { - //------------------------------------------------- - StringBuilder sb = new StringBuilder(); - - int modifiers = method.getModifiers(); - if ((modifiers & Modifier.STATIC) != 0) { - sb.append("static"); - } else { - sb.append("non-static"); - } - sb.append(" "); - Class returnType = method.getReturnType(); - sb.append(getParameterTypeName(returnType, versionedClassToNameMap)); - sb.append(" "); - sb.append(method.getName()); - Class[] parameterTypes = method.getParameterTypes(); - sb.append("("); - boolean isFirst = true; - for (Class parameterType : parameterTypes) { - if (!isFirst) { - sb.append(", "); - } - sb.append(getParameterTypeName(parameterType, versionedClassToNameMap)); - isFirst = false; - } - sb.append(")"); - Class[] exceptionsThrown = method.getExceptionTypes(); - if (exceptionsThrown.length > 0) { - sb.append(" throws "); - isFirst = true; - for (Class exceptionThrown : exceptionsThrown) { - if (!isFirst) { - sb.append(", "); - } - sb.append(exceptionThrown.getSimpleName()); - isFirst = false; - } - } - - return sb.toString(); - //------------------------------------------------- - } - - public String getFieldKey(Field field, Map versionedClassToNameMap) throws IllegalAccessException { - //------------------------------------------------- - StringBuilder sb = new StringBuilder(); - - int modifiers = field.getModifiers(); - if ((modifiers & Modifier.STATIC) != 0) { - sb.append("static"); - } else { - sb.append("non-static"); - } - sb.append(" "); - Class fieldType = field.getType(); - sb.append(getParameterTypeName(fieldType, versionedClassToNameMap)); - sb.append(" "); - sb.append(field.getName()); - if ((modifiers & Modifier.STATIC) != 0) { - sb.append(" "); - sb.append(field.get(null)); - } - - return sb.toString(); - //------------------------------------------------- - } - - public Method[] onlyPublicMethods(Method[] methods) { - List resultList = new ArrayList(); - for (Method method : methods) { - if ((method.getModifiers() & Modifier.PUBLIC) != 0) { - resultList.add(method); - } - } - return resultList.toArray(new Method[0]); - } - - public Field[] onlyPublicFields(Field[] fields) { - List resultList = new ArrayList(); - for (Field field : fields) { - if ((field.getModifiers() & Modifier.PUBLIC) != 0) { - resultList.add(field); - } - } - return resultList.toArray(new Field[0]); - } - - public TreeSet getMethodKeySetForAnnotation(Method[] methods, Class annotationClass, - Map versionedClassToNameMap) - throws IllegalAccessException { - TreeSet result = new TreeSet(); - - for (Method method : methods) { - Annotation[] declaredAnnotations = method.getDeclaredAnnotations(); - boolean isFound = false; - for (Annotation declaredAnnotation : declaredAnnotations) { - if (declaredAnnotation.annotationType().equals(annotationClass)) { - isFound = true; - break; - } - } - if (!isFound) { - continue; - } - result.add(getMethodKey(method, versionedClassToNameMap)); - } - return result; - } - - public TreeSet getMethodKeySetExcludingAnnotations(Method[] methods, - List versionAnnotations, Map versionedClassToNameMap) - throws IllegalAccessException { - TreeSet result = new TreeSet(); - - for (Method method : methods) { - Annotation[] declaredAnnotations = method.getDeclaredAnnotations(); - boolean isFound = false; - for (Annotation declaredAnnotation : declaredAnnotations) { - for (Class versionAnnotation : versionAnnotations) { - if (declaredAnnotation.annotationType().equals(versionAnnotation)) { - isFound = true; - break; - } - } - if (isFound) { - break; - } - } - if (isFound) { - continue; - } - String methodKey = getMethodKey(method, versionedClassToNameMap); - if (!methodKey.equals("non-static int compareTo(Object)")) { - result.add(methodKey); - } - } - return result; - } - - public TreeSet getFieldKeySetForAnnotation(Field[] fields, - Class annotationClass, Map versionedClassToNameMap) - throws IllegalAccessException { - TreeSet result = new TreeSet(); - - for (Field field : fields) { - Annotation[] declaredAnnotations = field.getDeclaredAnnotations(); - boolean isFound = false; - for (Annotation declaredAnnotation : declaredAnnotations) { - if (declaredAnnotation.annotationType().equals(annotationClass)) { - isFound = true; - break; - } - } - if (!isFound) { - continue; - } - result.add(getFieldKey(field, versionedClassToNameMap)); - } - return result; - } - - public TreeSet getFieldKeySetExcludingAnnotations(Field[] fields, - List versionAnnotations, Map versionedClassToNameMap) - throws IllegalAccessException { - TreeSet result = new TreeSet(); - - for (Field field : fields) { - Annotation[] declaredAnnotations = field.getDeclaredAnnotations(); - boolean isFound = false; - for (Annotation declaredAnnotation : declaredAnnotations) { - for (Class versionAnnotation : versionAnnotations) { - if (declaredAnnotation.annotationType().equals(versionAnnotation)) { - isFound = true; - break; - } - } - if (isFound) { - break; - } - } - if (isFound) { - continue; - } - result.add(getFieldKey(field, versionedClassToNameMap)); - } - return result; - } - - // For now, olderClass has 1 version and newerClass 2 versions... - public void doVerifyVersions( - Class olderClass, Class olderVersionClass, - Class newerClass, Class newerVersionClass, - Map versionedClassToNameMap) throws IllegalAccessException { - - List olderVersionClasses = new ArrayList(); - olderVersionClasses.add(olderVersionClass); - - List newerVersionClasses = new ArrayList(); - newerVersionClasses.add(olderVersionClass); - newerVersionClasses.add(newerVersionClass); - - //---------------------------------------------------------------------------------------------- - Method[] olderMethods = onlyPublicMethods(olderClass.getDeclaredMethods()); - TreeSet olderMethodSet = - getMethodKeySetForAnnotation(olderMethods, olderVersionClass, versionedClassToNameMap); - - TreeSet olderNoMethodAnnotationsSet = - getMethodKeySetExcludingAnnotations(olderMethods, olderVersionClasses, versionedClassToNameMap); - - Field[] olderFields = onlyPublicFields(olderClass.getFields()); - TreeSet olderFieldSet = - getFieldKeySetForAnnotation(olderFields, olderVersionClass, versionedClassToNameMap); - - TreeSet olderNoFieldAnnotationsSet = - getFieldKeySetExcludingAnnotations(olderFields, olderVersionClasses, versionedClassToNameMap); - //---------------------------------------------------------------------------------------------- - - Method[] newerMethods = onlyPublicMethods(newerClass.getDeclaredMethods()); - TreeSet newerMethodSetV1 = - getMethodKeySetForAnnotation(newerMethods, olderVersionClass, versionedClassToNameMap); - TreeSet newerMethodSetV2 = - getMethodKeySetForAnnotation(newerMethods, newerVersionClass, versionedClassToNameMap); - - TreeSet newerNoMethodAnnotationsSetV1andV2 = - getMethodKeySetExcludingAnnotations(newerMethods, newerVersionClasses, versionedClassToNameMap); - - Field[] newerFields = onlyPublicFields(newerClass.getFields()); - // doDisplayFields(newerFields, newerClass); - TreeSet newerFieldSetV1 = - getFieldKeySetForAnnotation(newerFields, olderVersionClass, versionedClassToNameMap); - TreeSet newerFieldSetV2 = - getFieldKeySetForAnnotation(newerFields, newerVersionClass, versionedClassToNameMap); - - TreeSet newerNoFieldAnnotationsSetV1andV2 = - getFieldKeySetExcludingAnnotations(newerFields, newerVersionClasses, versionedClassToNameMap); - - //---------------------------------------------------------------------------------------------- - // VALIDATION - //---------------------------------------------------------------------------------------------- - - // No version annotation? - if (olderNoMethodAnnotationsSet.size() != 0) { - Assert.assertTrue("Class " + olderClass.getSimpleName() + " has 1 or more public methods without a version V1 annotation " + - olderNoMethodAnnotationsSet.toString(), false); - } - if (olderNoFieldAnnotationsSet.size() != 0) { - Assert.assertTrue("Class " + olderClass.getSimpleName() + " has 1 or more public fields without a version V1 annotation " + - olderNoFieldAnnotationsSet.toString(), false); - } - if (newerNoMethodAnnotationsSetV1andV2.size() != 0) { - Assert.assertTrue("Class " + newerClass.getSimpleName() + " has 1 or more public methods without a version V1 or V2 annotation " + - newerNoMethodAnnotationsSetV1andV2.toString(), false); - } - if (newerNoFieldAnnotationsSetV1andV2.size() != 0) { - Assert.assertTrue("Class " + newerClass.getSimpleName() + " has 1 or more public methods without a version V1 or V2 annotation " + - newerNoFieldAnnotationsSetV1andV2.toString(), false); - } - - // Do the V1 methods of older and newer match? - if (!olderMethodSet.equals(newerMethodSetV1)) { - TreeSet leftCopy = new TreeSet(olderMethodSet); - leftCopy.removeAll(newerMethodSetV1); - TreeSet rightCopy = new TreeSet(newerMethodSetV1); - rightCopy.removeAll(olderMethodSet); - Assert.assertTrue("Class " + olderClass.getSimpleName() + " and class " + newerClass.getSimpleName() + " methods are different for V1 " + - leftCopy.toString() + " " + rightCopy.toString(), false); - } - - // Do the V1 fields of older and newer match? - if (!olderFieldSet.equals(newerFieldSetV1)) { - TreeSet leftCopy = new TreeSet(olderFieldSet); - leftCopy.removeAll(newerFieldSetV1); - TreeSet rightCopy = new TreeSet(newerFieldSetV1); - rightCopy.removeAll(olderFieldSet); - Assert.assertTrue("Class " + olderClass.getSimpleName() + " and class " + newerClass.getSimpleName() + " fields are different for V1 " + - leftCopy.toString() + " " + rightCopy.toString(), false); - } - } -} diff --git a/storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorTxnList.java b/storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorTxnList.java deleted file mode 100644 index 867b652116101fb784ccb30a267a6f4306761cd1..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorTxnList.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common; - -import org.junit.Assert; -import org.junit.Test; - -import java.util.BitSet; - -public class TestValidCompactorTxnList { - - @Test - public void minTxnHigh() { - BitSet bitSet = new BitSet(2); - bitSet.set(0, 2); - ValidTxnList txns = new ValidCompactorTxnList(new long[]{3, 4}, bitSet, 2); - ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9); - Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp); - } - - @Test - public void maxTxnLow() { - BitSet bitSet = new BitSet(2); - bitSet.set(0, 2); - ValidTxnList txns = new ValidCompactorTxnList(new long[]{13, 14}, bitSet, 12); - ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9); - Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp); - } - - @Test - public void minTxnHighNoExceptions() { - ValidTxnList txns = new ValidCompactorTxnList(new long[0], new BitSet(), 5); - ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9); - Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp); - } - - @Test - public void maxTxnLowNoExceptions() { - ValidTxnList txns = new ValidCompactorTxnList(new long[0], new BitSet(), 15); - ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9); - Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp); - } - - @Test - public void exceptionsAllBelow() { - BitSet bitSet = new BitSet(2); - bitSet.set(0, 2); - ValidTxnList txns = new ValidCompactorTxnList(new long[]{3, 6}, bitSet, 3); - ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9); - Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp); - } - - @Test - public void exceptionsInMidst() { - BitSet bitSet = new BitSet(1); - bitSet.set(0, 1); - ValidTxnList txns = new ValidCompactorTxnList(new long[]{8}, bitSet, 7); - ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9); - Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp); - } - @Test - public void exceptionsAbveHighWaterMark() { - BitSet bitSet = new BitSet(4); - bitSet.set(0, 4); - ValidTxnList txns = new ValidCompactorTxnList(new long[]{8, 11, 17, 29}, bitSet, 15); - Assert.assertArrayEquals("", new long[]{8, 11}, txns.getInvalidTransactions()); - ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9); - Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp); - rsp = txns.isTxnRangeValid(12, 16); - Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp); - } - - @Test - public void writeToString() { - BitSet bitSet = new BitSet(4); - bitSet.set(0, 4); - ValidTxnList txns = new ValidCompactorTxnList(new long[]{7, 9, 10, Long.MAX_VALUE}, bitSet, 8); - Assert.assertEquals("8:" + Long.MAX_VALUE + "::7", txns.writeToString()); - txns = new ValidCompactorTxnList(); - Assert.assertEquals(Long.toString(Long.MAX_VALUE) + ":" + Long.MAX_VALUE + "::", txns.writeToString()); - txns = new ValidCompactorTxnList(new long[0], new BitSet(), 23); - Assert.assertEquals("23:" + Long.MAX_VALUE + "::", txns.writeToString()); - } - - @Test - public void readFromString() { - ValidCompactorTxnList txns = new ValidCompactorTxnList("37:" + Long.MAX_VALUE + "::7,9,10"); - Assert.assertEquals(37L, txns.getHighWatermark()); - Assert.assertNull(txns.getMinOpenTxn()); - Assert.assertArrayEquals(new long[]{7L, 9L, 10L}, txns.getInvalidTransactions()); - txns = new ValidCompactorTxnList("21:" + Long.MAX_VALUE + ":"); - Assert.assertEquals(21L, txns.getHighWatermark()); - Assert.assertNull(txns.getMinOpenTxn()); - Assert.assertEquals(0, txns.getInvalidTransactions().length); - } - - @Test - public void testAbortedTxn() throws Exception { - ValidCompactorTxnList txnList = new ValidCompactorTxnList("5:4::1,2,3"); - Assert.assertEquals(5L, txnList.getHighWatermark()); - Assert.assertEquals(4, txnList.getMinOpenTxn().longValue()); - Assert.assertArrayEquals(new long[]{1L, 2L, 3L}, txnList.getInvalidTransactions()); - } - - @Test - public void testAbortedRange() throws Exception { - ValidCompactorTxnList txnList = new ValidCompactorTxnList("11:4::5,6,7,8"); - ValidTxnList.RangeResponse rsp = txnList.isTxnRangeAborted(1L, 3L); - Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp); - rsp = txnList.isTxnRangeAborted(9L, 10L); - Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp); - rsp = txnList.isTxnRangeAborted(6L, 7L); - Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp); - rsp = txnList.isTxnRangeAborted(4L, 6L); - Assert.assertEquals(ValidTxnList.RangeResponse.SOME, rsp); - rsp = txnList.isTxnRangeAborted(6L, 13L); - Assert.assertEquals(ValidTxnList.RangeResponse.SOME, rsp); - } -} diff --git a/storage-api/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java b/storage-api/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java deleted file mode 100644 index 00ee820b2ebc4111b4b43fd48e8dd75978ee0754..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common; - -import junit.framework.Assert; -import org.apache.hadoop.conf.Configuration; -import org.junit.Test; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.util.BitSet; - -/** - * Tests for {@link ValidReadTxnList} - */ -public class TestValidReadTxnList { - - @Test - public void noExceptions() throws Exception { - ValidTxnList txnList = new ValidReadTxnList(new long[0], new BitSet(), 1, Long.MAX_VALUE); - String str = txnList.writeToString(); - Assert.assertEquals("1:" + Long.MAX_VALUE + "::", str); - ValidTxnList newList = new ValidReadTxnList(); - newList.readFromString(str); - Assert.assertTrue(newList.isTxnValid(1)); - Assert.assertFalse(newList.isTxnValid(2)); - } - - @Test - public void exceptions() throws Exception { - ValidTxnList txnList = new ValidReadTxnList(new long[]{2L,4L}, new BitSet(), 5, 4L); - String str = txnList.writeToString(); - Assert.assertEquals("5:4:2,4:", str); - ValidTxnList newList = new ValidReadTxnList(); - newList.readFromString(str); - Assert.assertTrue(newList.isTxnValid(1)); - Assert.assertFalse(newList.isTxnValid(2)); - Assert.assertTrue(newList.isTxnValid(3)); - Assert.assertFalse(newList.isTxnValid(4)); - Assert.assertTrue(newList.isTxnValid(5)); - Assert.assertFalse(newList.isTxnValid(6)); - } - - @Test - public void longEnoughToCompress() throws Exception { - long[] exceptions = new long[1000]; - for (int i = 0; i < 1000; i++) exceptions[i] = i + 100; - ValidTxnList txnList = new ValidReadTxnList(exceptions, new BitSet(), 2000, 900); - String str = txnList.writeToString(); - ValidTxnList newList = new ValidReadTxnList(); - newList.readFromString(str); - for (int i = 0; i < 100; i++) Assert.assertTrue(newList.isTxnValid(i)); - for (int i = 100; i < 1100; i++) Assert.assertFalse(newList.isTxnValid(i)); - for (int i = 1100; i < 2001; i++) Assert.assertTrue(newList.isTxnValid(i)); - Assert.assertFalse(newList.isTxnValid(2001)); - } - - @Test - public void readWriteConfig() throws Exception { - long[] exceptions = new long[1000]; - for (int i = 0; i < 1000; i++) exceptions[i] = i + 100; - ValidTxnList txnList = new ValidReadTxnList(exceptions, new BitSet(), 2000, 900); - String str = txnList.writeToString(); - Configuration conf = new Configuration(); - conf.set(ValidTxnList.VALID_TXNS_KEY, str); - File tmpFile = File.createTempFile("TestValidTxnImpl", "readWriteConfig"); - DataOutputStream out = new DataOutputStream(new FileOutputStream(tmpFile)); - conf.write(out); - out.close(); - DataInputStream in = new DataInputStream(new FileInputStream(tmpFile)); - Configuration newConf = new Configuration(); - newConf.readFields(in); - Assert.assertEquals(str, newConf.get(ValidTxnList.VALID_TXNS_KEY)); - } - - @Test - public void testAbortedTxn() throws Exception { - long[] exceptions = {2L, 4L, 6L, 8L, 10L}; - BitSet bitSet = new BitSet(exceptions.length); - bitSet.set(0); // mark txn "2L" aborted - bitSet.set(3); // mark txn "8L" aborted - ValidTxnList txnList = new ValidReadTxnList(exceptions, bitSet, 11, 4L); - String str = txnList.writeToString(); - Assert.assertEquals("11:4:4,6,10:2,8", str); - Assert.assertTrue(txnList.isTxnAborted(2L)); - Assert.assertFalse(txnList.isTxnAborted(4L)); - Assert.assertFalse(txnList.isTxnAborted(6L)); - Assert.assertTrue(txnList.isTxnAborted(8L)); - Assert.assertFalse(txnList.isTxnAborted(10L)); - } -} diff --git a/storage-api/src/test/org/apache/hadoop/hive/common/io/TestDiskRangeList.java b/storage-api/src/test/org/apache/hadoop/hive/common/io/TestDiskRangeList.java deleted file mode 100644 index a76c7ad7995e919334355f3b4372f057b91e1af8..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/common/io/TestDiskRangeList.java +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.io; - -import org.junit.Test; - -public class TestDiskRangeList { - @Test - public void testErrorConditions() throws Exception { - DiskRangeList d510 = new DiskRangeList(0, 10); - d510.insertPartBefore(new DiskRangeList(0, 5)); - DiskRangeList d1015 = d510.insertAfter(new DiskRangeList(10, 15)); - try { - d510.replaceSelfWith(d510); // The arg is self. - fail(); - } catch (AssertionError error) {} - DiskRangeList existing = new DiskRangeList(0, 10); - existing.insertPartBefore(new DiskRangeList(0, 5)); - try { - d510.replaceSelfWith(existing); // The arg is part of another list. - fail(); - } catch (AssertionError error) {} - try { - d510.replaceSelfWith(new DiskRangeList(4, 10)); // Not sequential with previous. - fail(); - } catch (AssertionError error) {} - try { - d510.replaceSelfWith(new DiskRangeList(5, 11)); // Not sequential with next. - fail(); - } catch (AssertionError error) {} - - - try { - d510.insertPartBefore(d510); // The arg is self. - fail(); - } catch (AssertionError error) {} - existing = new DiskRangeList(5, 7); - existing.insertPartBefore(new DiskRangeList(5, 6)); - try { - d510.insertPartBefore(existing); // The arg is part of another list. - fail(); - } catch (AssertionError error) {} - try { - d510.insertPartBefore(new DiskRangeList(3, 4)); // Not a part. - fail(); - } catch (AssertionError error) {} - try { - d510.insertPartBefore(new DiskRangeList(4, 6)); // Not sequential with previous. - fail(); - } catch (AssertionError error) {} - - - try { - d510.insertAfter(d510); // The arg is self. - fail(); - } catch (AssertionError error) {} - existing = new DiskRangeList(15, 20); - existing.insertAfter(new DiskRangeList(20, 25)); - try { - d1015.insertAfter(existing); // The arg is part of another list. - fail(); - } catch (AssertionError error) {} - try { - d1015.insertAfter(new DiskRangeList(14, 20)); // Not sequential. - fail(); - } catch (AssertionError error) {} - d1015.insertAfter(new DiskRangeList(20, 25)); - try { - d1015.insertAfter(new DiskRangeList(15, 21)); // Not sequential with next. - fail(); - } catch (AssertionError error) {} - try { - d1015.insertPartAfter(new DiskRangeList(16, 20)); // Not a part. - fail(); - } catch (AssertionError error) {} - try { - d1015.insertPartAfter(new DiskRangeList(9, 11)); // Not a part. - fail(); - } catch (AssertionError error) {} - - try { - d1015.setEnd(21); // Not sequential with next. - fail(); - } catch (AssertionError error) {} - } - - private void fail() throws Exception { - throw new Exception(); // Don't use Assert.fail, we are catching assertion errors. - } -} diff --git a/storage-api/src/test/org/apache/hadoop/hive/common/type/HiveDecimalTestBase.java b/storage-api/src/test/org/apache/hadoop/hive/common/type/HiveDecimalTestBase.java deleted file mode 100644 index 9550b948ce3d2d900d143da708f9667cc0bb65e3..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/common/type/HiveDecimalTestBase.java +++ /dev/null @@ -1,568 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -import java.util.Random; -import java.math.BigDecimal; -import java.math.BigInteger; - -import org.apache.hadoop.hive.common.type.RandomTypeUtil; - -public class HiveDecimalTestBase { - - public static int POUND_FACTOR = 1000; - - public static enum BigDecimalFlavor { - NORMAL_RANGE, - FRACTIONS_ONLY, - NEGATIVE_SCALE, - LONG_TAIL - } - - public static enum BigDecimalPairFlavor { - RANDOM, - NEAR, - INVERSE - } - - public BigDecimal randHiveBigDecimal(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - switch (bigDecimalFlavor) { - case NORMAL_RANGE: - return randHiveBigDecimalNormalRange(r, digitAlphabet); - case FRACTIONS_ONLY: - return randHiveBigDecimalFractionsOnly(r, digitAlphabet); - case NEGATIVE_SCALE: - return randHiveBigDecimalNegativeScale(r, digitAlphabet); - case LONG_TAIL: - return randHiveBigDecimalLongTail(r, digitAlphabet); - default: - throw new RuntimeException("Unexpected big decimal flavor " + bigDecimalFlavor); - } - } - - public BigDecimal[] randHiveBigDecimalPair(Random r, String digitAlphabet, - BigDecimalFlavor bigDecimalFlavor, BigDecimalPairFlavor bigDecimalPairFlavor) { - BigDecimal[] pair = new BigDecimal[2]; - BigDecimal bigDecimal1 = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - pair[0] = bigDecimal1; - - BigDecimal bigDecimal2; - switch (bigDecimalPairFlavor) { - case RANDOM: - bigDecimal2 = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - break; - case NEAR: - bigDecimal2 = randHiveBigDecimalNear(r, bigDecimal1); - break; - case INVERSE: - bigDecimal2 = randHiveBigDecimalNear(r, bigDecimal1); - break; - default: - throw new RuntimeException("Unexpected big decimal pair flavor " + bigDecimalPairFlavor); - } - pair[1] = bigDecimal2; - return pair; - } - - public BigDecimal randHiveBigDecimalNormalRange(Random r, String digitAlphabet) { - String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(38)); - BigInteger bigInteger = new BigInteger(digits); - boolean negated = false; - if (r.nextBoolean()) { - bigInteger = bigInteger.negate(); - negated = true; - } - int scale = 0 + r.nextInt(38 + 1); - return new BigDecimal(bigInteger, scale); - } - - public BigDecimal randHiveBigDecimalNegativeScale(Random r, String digitAlphabet) { - String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(38)); - BigInteger bigInteger = new BigInteger(digits); - boolean negated = false; - if (r.nextBoolean()) { - bigInteger = bigInteger.negate(); - negated = true; - } - int scale = 0 + (r.nextBoolean() ? 0 : r.nextInt(38 + 1)); - if (r.nextBoolean()) { - scale = -scale; - } - return new BigDecimal(bigInteger, scale); - } - - public BigDecimal randHiveBigDecimalLongTail(Random r, String digitAlphabet) { - int scale = 0 + r.nextInt(38 + 20); - final int maxDigits = 38 + (scale == 0 ? 0 : 20); - String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(maxDigits)); - BigInteger bigInteger = new BigInteger(digits); - boolean negated = false; - if (r.nextBoolean()) { - bigInteger = bigInteger.negate(); - negated = true; - } - return new BigDecimal(bigInteger, scale); - } - - public BigDecimal randHiveBigDecimalFractionsOnly(Random r, String digitAlphabet) { - int scale = 1 + r.nextInt(38 + 1); - String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(scale)); - BigInteger bigInteger = new BigInteger(digits); - boolean negated = false; - if (r.nextBoolean()) { - bigInteger = bigInteger.negate(); - negated = true; - } - return new BigDecimal(bigInteger, scale); - } - - public BigDecimal randHiveBigDecimalNear(Random r, BigDecimal bigDecimal) { - - int scale = bigDecimal.scale(); - int delta = r.nextInt(10); - if (r.nextBoolean()) { - return bigDecimal.add(new BigDecimal(BigInteger.valueOf(delta), scale)); - } else { - return bigDecimal.subtract(new BigDecimal(BigInteger.valueOf(delta), scale)); - } - } - - public BigDecimal randHiveBigDecimalInverse(Random r, BigDecimal bigDecimal) { - if (bigDecimal.signum() == 0) { - return bigDecimal; - } - return BigDecimal.ONE.divide(bigDecimal); - } - - public BigInteger randHiveBigInteger(Random r, String digitAlphabet) { - String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(38)); - BigInteger bigInteger = new BigInteger(digits); - boolean negated = false; - if (r.nextBoolean()) { - bigInteger = bigInteger.negate(); - negated = true; - } - return bigInteger; - } - - public boolean isTenPowerBug(String string) { - // // System.out.println("TEST_IS_TEN_TO_38_STRING isTenPowerBug " + string); - if (string.charAt(0) == '-') { - string = string.substring(1); - } - int index = string.indexOf('.'); - if (index != -1) { - if (index == 0) { - string = string.substring(1); - } else { - string = string.substring(0, index) + string.substring(index + 1); - } - } - // // System.out.println("TEST_IS_TEN_TO_38_STRING isTenPowerBug " + string); - return string.equals("100000000000000000000000000000000000000"); - } - - //------------------------------------------------------------------------------------------------ - - public static String[] specialDecimalStrings = new String[] { - "0", - "1", - "-1", - "10", - "-10", - "100", - "-100", - "127", // Byte.MAX_VALUE - "127.1", - "127.0008", - "127.49", - "127.5", - "127.9999999999999999999", - "-127", - "-127.1", - "-127.0008", - "-127.49", - "-127.5", - "-127.999999", - "128", - "128.1", - "128.0008", - "128.49", - "128.5", - "128.9999999999999999999", - "-128", // Byte.MIN_VALUE - "-128.1", - "-128.0008", - "-128.49", - "-128.5", - "-128.999", - "129", - "129.1", - "-129", - "-129.1", - "1000", - "-1000", - "10000", - "-10000", - "32767", // Short.MAX_VALUE - "32767.1", - "32767.0008", - "32767.49", - "32767.5", - "32767.99999999999", - "-32767", - "-32767.1", - "-32767.0008", - "-32767.49", - "-32767.5", - "-32767.9", - "32768", - "32768.1", - "32768.0008", - "32768.49", - "32768.5", - "32768.9999999999", - "-32768", // Short.MIN_VALUE - "-32768.1", - "-32768.0008", - "-32768.49", - "-32768.5", - "-32768.9999999", - "32769", - "32769.1", - "-32769", - "-32769.1", - "100000", - "-100000", - "1000000", - "-1000000", - "10000000", - "-10000000", - "100000000", - "99999999", // 10^8 - 1 - "-99999999", - "-100000000", - "1000000000", - "-1000000000", - "2147483647", // Integer.MAX_VALUE - "2147483647.1", - "2147483647.0008", - "2147483647.49", - "2147483647.5", - "2147483647.9999999999", - "-2147483647", - "-2147483647.1", - "-2147483647.0008", - "-2147483647.49", - "-2147483647.5", - "-2147483647.9999999999999999999", - "2147483648", - "2147483648.1", - "2147483648.0008", - "2147483648.49", - "2147483648.5", - "2147483648.9", - "-2147483648", // Integer.MIN_VALUE - "-2147483648.1", - "-2147483648.0008", - "-2147483648.49", - "-2147483648.5", - "-2147483648.999", - "2147483649", - "2147483649.1", - "-2147483649", - "-2147483649.1", - "10000000000", - "-10000000000", - "100000000000", - "-100000000000", - "1000000000000", - "-1000000000000", - "10000000000000", - "-10000000000000", - "100000000000000", - "-100000000000000", - "999999999999999", - "-999999999999999", - "1000000000000000", // 10^15 - "-1000000000000000", - "9999999999999999", // 10^16 - 1 - "-9999999999999999", - "10000000000000000", // 10^16 - "-10000000000000000", - "99999999999999999", // 10^17 - 1 - "-99999999999999999", - "100000000000000000", - "-100000000000000000", - "999999999999999999", // 10^18 - 1 - "-999999999999999999", - "123456789012345678", - "-123456789012345678", - "1000000000000000000", - "-1000000000000000000", - "9223372036854775807", // Long.MAX_VALUE - "9223372036854775807.1", - "9223372036854775807.0008", - "9223372036854775807.49", - "9223372036854775807.5", - "9223372036854775807.9", - "-9223372036854775807", - "-9223372036854775807.1", - "-9223372036854775807.0008", - "-9223372036854775807.49", - "-9223372036854775807.5", - "-9223372036854775807.9999999999999999999", - "-9223372036854775808", - "-9223372036854775808.1", - "9223372036854775808", - "9223372036854775808.1", - "9223372036854775808.0008", - "9223372036854775808.49", - "9223372036854775808.5", - "9223372036854775808.9", - "9223372036854775809", - "9223372036854775809.1", - "-9223372036854775808", // Long.MIN_VALUE - "-9223372036854775808.1", - "-9223372036854775808.0008", - "-9223372036854775808.49", - "-9223372036854775808.5", - "-9223372036854775808.9999999", - "9223372036854775809", - "9223372036854775809.1", - "-9223372036854775809", - "-9223372036854775809.1", - "10000000000000000000000000000000", // 10^31 - "-10000000000000000000000000000000", - "99999999999999999999999999999999", // 10^32 - 1 - "-99999999999999999999999999999999", - "100000000000000000000000000000000", // 10^32 - "-100000000000000000000000000000000", - "10000000000000000000000000000000000000", // 10^37 - "-10000000000000000000000000000000000000", - "99999999999999999999999999999999999999", // 10^38 - 1 - "-99999999999999999999999999999999999999", - "100000000000000000000000000000000000000", // 10^38 - "-100000000000000000000000000000000000000", - "1000000000000000000000000000000000000000", // 10^39 - "-1000000000000000000000000000000000000000", - - "18446744073709551616", // Unsigned 64 max. - "-18446744073709551616", - "340282366920938463463374607431768211455", // 2^128 - 1 - "-340282366920938463463374607431768211455", - - "0.999999999999999", - "-0.999999999999999", - "0.0000000000000001", // 10^-15 - "-0.0000000000000001", - "0.9999999999999999", - "-0.9999999999999999", - "0.00000000000000001", // 10^-16 - "-0.00000000000000001", - "0.99999999999999999", - "-0.99999999999999999", - "0.999999999999999999", // 10^-18 - "-0.999999999999999999", - "0.00000000000000000000000000000001", // 10^-31 - "-0.00000000000000000000000000000001", - "0.99999999999999999999999999999999", // 10^-32 + 1 - "-0.99999999999999999999999999999999", - "0.000000000000000000000000000000001", // 10^-32 - "-0.000000000000000000000000000000001", - "0.00000000000000000000000000000000000001", // 10^-37 - "-0.00000000000000000000000000000000000001", - "0.99999999999999999999999999999999999999", // 10^-38 + 1 - "-0.99999999999999999999999999999999999999", - "0.000000000000000000000000000000000000001", // 10^-38 - "-0.000000000000000000000000000000000000001", - "0.0000000000000000000000000000000000000001", // 10^-39 - "-0.0000000000000000000000000000000000000001", - "0.0000000000000000000000000000000000000005", // 10^-39 (rounds) - "-0.0000000000000000000000000000000000000005", - "0.340282366920938463463374607431768211455", // (2^128 - 1) * 10^-39 - "-0.340282366920938463463374607431768211455", - "0.000000000000000000000000000000000000001", // 10^-38 - "-0.000000000000000000000000000000000000001", - "0.000000000000000000000000000000000000005", // 10^-38 - "-0.000000000000000000000000000000000000005", - - "234.79", - "342348.343", - "12.25", - "-12.25", - "72057594037927935", // 2^56 - 1 - "-72057594037927935", - "72057594037927936", // 2^56 - "-72057594037927936", - "5192296858534827628530496329220095", // 2^56 * 2^56 - 1 - "-5192296858534827628530496329220095", - "5192296858534827628530496329220096", // 2^56 * 2^56 - "-5192296858534827628530496329220096", - - "54216721532321902598.70", - "-906.62545207002374150309544832320", - "-0.0709351061072", - "1460849063411925.53", - "8.809130E-33", - "-4.0786300706013636202E-20", - "-3.8823936518E-1", - "-3.8823936518E-28", - "-3.8823936518E-29", - "598575157855521918987423259.94094", - "299999448432.001342152474197", - "1786135888657847525803324040144343378.09799306448796128931113691624", // More than 38 digits. - "-1786135888657847525803324040144343378.09799306448796128931113691624", - "57847525803324040144343378.09799306448796128931113691624", - "0.999999999999999999990000", - "005.34000", - "1E-90", - - "0.4", - "-0.4", - "0.5", - "-0.5", - "0.6", - "-0.6", - "1.4", - "-1.4", - "1.5", - "-1.5", - "1.6", - "-1.6", - "2.4", - "-2.4", - "2.49", - "-2.49", - "2.5", - "-2.5", - "2.51", - "-2.51", - "-2.5", - "2.6", - "-2.6", - "3.00001415926", - "0.00", - "-12.25", - "234.79" - }; - - public static BigDecimal[] specialBigDecimals = stringArrayToBigDecimals(specialDecimalStrings); - - // decimal_1_1.txt - public static String[] decimal_1_1_txt = { - "0.0", - "0.0000", - ".0", - "0.1", - "0.15", - "0.9", - "0.94", - "0.99", - "0.345", - "1.0", - "1", - "0", - "00", - "22", - "1E-9", - "-0.0", - "-0.0000", - "-.0", - "-0.1", - "-0.15", - "-0.9", - "-0.94", - "-0.99", - "-0.345", - "-1.0", - "-1", - "-0", - "-00", - "-22", - "-1E-9" - }; - - // kv7.txt KEYS - public static String[] kv7_txt_keys = { - "-4400", - "1E+99", - "1E-99", - "0", - "100", - "10", - "1", - "0.1", - "0.01", - "200", - "20", - "2", - "0", - "0.2", - "0.02", - "0.3", - "0.33", - "0.333", - "-0.3", - "-0.33", - "-0.333", - "1.0", - "2", - "3.14", - "-1.12", - "-1.12", - "-1.122", - "1.12", - "1.122", - "124.00", - "125.2", - "-1255.49", - "3.14", - "3.14", - "3.140", - "0.9999999999999999999999999", - "-1234567890.1234567890", - "1234567890.1234567800" - }; - - public static String standardAlphabet = "0123456789"; - - public static String[] sparseAlphabets = new String[] { - - "0000000000000000000000000000000000000003", - "0000000000000000000000000000000000000009", - "0000000000000000000000000000000000000001", - "0000000000000000000003", - "0000000000000000000009", - "0000000000000000000001", - "0000000000091", - "000000000005", - "9", - "5555555555999999999000000000000001111111", - "24680", - "1" - }; - - public static BigDecimal[] stringArrayToBigDecimals(String[] strings) { - BigDecimal[] result = new BigDecimal[strings.length]; - for (int i = 0; i < strings.length; i++) { - result[i] = new BigDecimal(strings[i]); - } - return result; - } -} \ No newline at end of file diff --git a/storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java b/storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java deleted file mode 100644 index d11f41c65f56487e1e267130a027d196d332908a..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java +++ /dev/null @@ -1,3477 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -import java.sql.Timestamp; -import java.util.Random; -import java.util.Arrays; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.math.BigDecimal; -import java.math.BigInteger; - -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritableV1; -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; -import org.apache.hadoop.hive.common.type.RandomTypeUtil; -import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; -import org.apache.hadoop.hive.ql.util.TimestampUtils; - -import org.junit.*; - -import static org.junit.Assert.*; - -public class TestHiveDecimal extends HiveDecimalTestBase { - - @Test - public void testInvalidStringInput() { - - HiveDecimalV1 oldDec; - HiveDecimalV1 resultOldDec; - HiveDecimal dec; - HiveDecimal resultDec; - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("-"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("-"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("+"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("+"); - Assert.assertTrue(dec == null); - - // Naked dot. - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("."); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("."); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("-."); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("-."); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("+."); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("+."); - Assert.assertTrue(dec == null); - - // Naked E/e. - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("E"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("E"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(".E"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create(".E"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("-E"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("-E"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("+E"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("+E"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("e"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("e"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(".e"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create(".e"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("-e"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("-e"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("+e"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("+e"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("error"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("error"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("0x0"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("0x0"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("0e"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("0e"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("7e"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("7e"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("233e-"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("233e-"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("32e+"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("32e+"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(".0e"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create(".0e"); - Assert.assertTrue(dec == null); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(".4e"); - Assert.assertTrue(oldDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create(".4e"); - Assert.assertTrue(dec == null); - } - - @Test - public void testVariousCases() { - - HiveDecimalV1 oldDec; - HiveDecimalV1 resultOldDec; - HiveDecimal dec; - HiveDecimal resultDec; - - BigDecimal bigDecimal = new BigDecimal("-99999999999999999999999999999999999999.99999999999999999"); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(bigDecimal); - Assert.assertEquals("-100000000000000000000000000000000000000", oldDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(bigDecimal); - Assert.assertTrue(dec == null); - - // One less integer digit... - bigDecimal = new BigDecimal("-9999999999999999999999999999999999999.99999999999999999"); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(bigDecimal); - Assert.assertEquals("-10000000000000000000000000000000000000", oldDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(bigDecimal); - Assert.assertEquals("-10000000000000000000000000000000000000", dec.toString()); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("101"); - resultOldDec = HiveDecimalV1.enforcePrecisionScale(oldDec, 10, 0); - Assert.assertEquals("101", resultOldDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create("101"); - resultDec = HiveDecimal.enforcePrecisionScale(dec, 10, 0); - Assert.assertEquals("101", resultDec.toString()); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("1"); - resultOldDec = oldDec.scaleByPowerOfTen(-99); - Assert.assertEquals("0", resultOldDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create("1"); - resultDec = dec.scaleByPowerOfTen(-99); - Assert.assertEquals("0", resultDec.toString()); - } - - @Test - public void testCreateFromBigIntegerRounding() { - - BigInteger bigInt; - HiveDecimalV1 oldDec; - HiveDecimal dec; - - // 1786135888657847525803324040144343378.09799306448796128931113691624 - bigInt = new BigInteger( - "178613588865784752580332404014434337809799306448796128931113691624"); - Assert.assertEquals("178613588865784752580332404014434337809799306448796128931113691624", bigInt.toString()); - // 12345678901234567890123456789012345678 - // 1 2 3 - // 12345678901234567890123456789 - dec = HiveDecimal.create(bigInt, 29); - Assert.assertEquals("1786135888657847525803324040144343378.1", dec.toString()); - - // 8.090000000000000000000000000000000000000123456 - bigInt = new BigInteger( - "8090000000000000000000000000000000000000123456"); - // 123456789012345678901234567890123456789012345 - // 1 2 3 4 - Assert.assertEquals("8090000000000000000000000000000000000000123456", bigInt.toString()); - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(bigInt, 45); - Assert.assertEquals("8.09", oldDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(bigInt, 45); - Assert.assertEquals("8.09", dec.toString()); - - // 99999999.99999999999999999999999999999949999 - // MAX_DECIMAL 9's WITH NO ROUND (longer than 38 digits) - bigInt = new BigInteger( - "9999999999999999999999999999999999999949999"); - // 12345678901234567890123456789012345678 - // 1 2 3 - // 99999999.99999999999999999999999999999949999 - Assert.assertEquals("9999999999999999999999999999999999999949999", bigInt.toString()); - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(bigInt, 35); - Assert.assertEquals("99999999.999999999999999999999999999999", oldDec.toString()); - //--------------------------------------------------- - // Without the round, this conversion fails. - dec = HiveDecimal.create(bigInt, 35); - Assert.assertEquals("99999999.999999999999999999999999999999", dec.toString()); - - // MAX_DECIMAL 9's WITH ROUND. - bigInt = new BigInteger( - "9999999999999999999999999999999999999979999"); - // 12346678.901234667890123466789012346678 - // 1 2 3 - Assert.assertEquals("9999999999999999999999999999999999999979999", bigInt.toString()); - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(bigInt, 35); - Assert.assertEquals("100000000", oldDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(bigInt, 35); - Assert.assertEquals("100000000", dec.toString()); - } - - @Test - public void testCreateFromBigDecimal() { - - BigDecimal bigDec; - HiveDecimalV1 oldDec; - HiveDecimal dec; - - bigDec = new BigDecimal("0"); - Assert.assertEquals("0", bigDec.toString()); - dec = HiveDecimal.create(bigDec); - Assert.assertEquals("0", dec.toString()); - - bigDec = new BigDecimal("1"); - Assert.assertEquals("1", bigDec.toString()); - dec = HiveDecimal.create(bigDec); - Assert.assertEquals("1", dec.toString()); - - bigDec = new BigDecimal("0.999"); - Assert.assertEquals("0.999", bigDec.toString()); - dec = HiveDecimal.create(bigDec); - Assert.assertEquals("0.999", dec.toString()); - - // HiveDecimal suppresses trailing zeroes. - bigDec = new BigDecimal("0.9990"); - Assert.assertEquals("0.9990", bigDec.toString()); - dec = HiveDecimal.create(bigDec); - Assert.assertEquals("0.999", dec.toString()); - } - - @Test - public void testCreateFromBigDecimalRounding() { - - BigDecimal bigDec; - HiveDecimalV1 oldDec; - HiveDecimal dec; - - bigDec = new BigDecimal( - "1786135888657847525803324040144343378.09799306448796128931113691624"); - Assert.assertEquals("1786135888657847525803324040144343378.09799306448796128931113691624", bigDec.toString()); - // 1234567890123456789012345678901234567.8 - // 1 2 3 - // Without the round, this conversion fails. - dec = HiveDecimal.create(bigDec, false); - Assert.assertTrue(dec == null); - dec = HiveDecimal.create(bigDec, true); - Assert.assertEquals("1786135888657847525803324040144343378.1", dec.toString()); - - bigDec = new BigDecimal( - "8.090000000000000000000000000000000000000123456"); - // 1.23456789012345678901234567890123456789012345 - // 1 2 3 4 - Assert.assertEquals("8.090000000000000000000000000000000000000123456", bigDec.toString()); - //--------------------------------------------------- - HiveDecimalV1 oldDec4 = HiveDecimalV1.create(bigDec, false); - Assert.assertTrue(oldDec4 == null); - oldDec4 = HiveDecimalV1.create(bigDec, true); - Assert.assertEquals("8.09", oldDec4.toString()); - //--------------------------------------------------- - // Without the round, this conversion fails. - dec = HiveDecimal.create(bigDec, false); - Assert.assertTrue(dec == null); - dec = HiveDecimal.create(bigDec, true); - Assert.assertEquals("8.09", dec.toString()); - - // MAX_DECIMAL 9's WITH NO ROUND (longer than 38 digits) - bigDec = new BigDecimal( - "99999999.99999999999999999999999999999949999"); - // 12345678.901234567890123456789012345678 - // 1 2 3 - Assert.assertEquals("99999999.99999999999999999999999999999949999", bigDec.toString()); - //--------------------------------------------------- - HiveDecimalV1 oldDec5 = HiveDecimalV1.create(bigDec, false); - Assert.assertTrue(oldDec5 == null); - oldDec5 = HiveDecimalV1.create(bigDec, true); - Assert.assertEquals("99999999.999999999999999999999999999999", oldDec5.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(bigDec, false); - Assert.assertTrue(dec == null); - dec = HiveDecimal.create(bigDec, true); - Assert.assertEquals("99999999.999999999999999999999999999999", dec.toString()); - - // MAX_DECIMAL 9's WITH ROUND. - bigDec = new BigDecimal( - "99999999.99999999999999999999999999999979999"); - // 12346678.901234667890123466789012346678 - // 1 2 3 - Assert.assertEquals("99999999.99999999999999999999999999999979999", bigDec.toString()); - //--------------------------------------------------- - HiveDecimalV1 oldDec6 = HiveDecimalV1.create(bigDec, false); - Assert.assertTrue(oldDec6 == null); - oldDec6 = HiveDecimalV1.create(bigDec, true); - Assert.assertEquals("100000000", oldDec6.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(bigDec, false); - Assert.assertTrue(dec == null); - dec = HiveDecimal.create(bigDec, true); - Assert.assertEquals("100000000", dec.toString()); - } - - - @Test - public void testPrecisionScaleEnforcement() { - - HiveDecimalV1 oldDec; - HiveDecimalV1 oldResultDec; - - HiveDecimal dec; - HiveDecimal resultDec; - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("0.02538461538461538461538461538461538462"); - Assert.assertEquals("0.02538461538461538461538", - HiveDecimalV1.enforcePrecisionScale(oldDec, 38, 23).toString()); - //--------------------------------------------------- - dec = HiveDecimal.create("0.02538461538461538461538461538461538462"); - Assert.assertEquals("0.02538461538461538461538", - HiveDecimal.enforcePrecisionScale(dec, 38, 23).toString()); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("005.34000"); - Assert.assertEquals(oldDec.precision(), 3); // 1 integer digit; 2 fraction digits. - Assert.assertEquals(oldDec.scale(), 2); // Trailing zeroes are suppressed. - //--------------------------------------------------- - dec = HiveDecimal.create("005.34000"); - Assert.assertEquals(dec.precision(), 3); // 1 integer digit; 2 fraction digits. - Assert.assertEquals(dec.scale(), 2); // Trailing zeroes are suppressed. - - dec = HiveDecimal.create("178613588865784752580332404014434337809799306448796128931113691624"); - Assert.assertNull(dec); - - // Rounding numbers that increase int digits - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("9.5"); - Assert.assertEquals("10", - HiveDecimalV1.enforcePrecisionScale(oldDec, 2, 0).toString()); - Assert.assertNull( - HiveDecimalV1.enforcePrecisionScale(oldDec, 1, 0)); - oldDec = HiveDecimalV1.create("9.4"); - Assert.assertEquals("9", - HiveDecimalV1.enforcePrecisionScale(oldDec, 1, 0).toString()); - //--------------------------------------------------- - dec = HiveDecimal.create("9.5"); - Assert.assertEquals("10", - HiveDecimal.enforcePrecisionScale(dec, 2, 0).toString()); - Assert.assertNull( - HiveDecimal.enforcePrecisionScale(dec, 1, 0)); - dec = HiveDecimal.create("9.4"); - Assert.assertEquals("9", - HiveDecimal.enforcePrecisionScale(dec, 1, 0).toString()); - } - - @Test - public void testPrecisionScaleEnforcementEdgeCond() { - - // Since HiveDecimal now uses FastHiveDecimal which stores 16 decimal digits per long, - // lets test edge conditions here. - - HiveDecimal fifteenFractionalNinesDec = HiveDecimal.create("0.999999999999999"); - Assert.assertNotNull(fifteenFractionalNinesDec); - Assert.assertEquals("0.999999999999999", - HiveDecimal.enforcePrecisionScale(fifteenFractionalNinesDec, 15, 15).toString()); - - HiveDecimal sixteenFractionalNines = HiveDecimal.create("0.9999999999999999"); - Assert.assertNotNull(sixteenFractionalNines); - Assert.assertEquals("0.9999999999999999", - HiveDecimal.enforcePrecisionScale(sixteenFractionalNines, 16, 16).toString()); - - HiveDecimal seventeenFractionalNines = HiveDecimal.create("0.99999999999999999"); - Assert.assertNotNull(seventeenFractionalNines); - Assert.assertEquals("0.99999999999999999", - HiveDecimal.enforcePrecisionScale(seventeenFractionalNines, 17, 17).toString()); - - } - - @Test - public void testTrailingZeroRemovalAfterEnforcement() { - String decStr = "8.090000000000000000000000000000000000000123456"; - // 123456789012345678901234567890123456789012345 - // 1 2 3 4 - HiveDecimal dec = HiveDecimal.create(decStr); - Assert.assertEquals("8.09", dec.toString()); - } - - @Test - public void testMultiply() { - - // This multiply produces more than 38 digits --> overflow. - //--------------------------------------------------- - HiveDecimalV1 oldDec1 = HiveDecimalV1.create("0.00001786135888657847525803"); - HiveDecimalV1 oldDec2 = HiveDecimalV1.create("3.0000123456789"); - HiveDecimalV1 oldResult = oldDec1.multiply(oldDec2); - Assert.assertTrue(oldResult == null); - //--------------------------------------------------- - HiveDecimal dec1 = HiveDecimal.create("0.00001786135888657847525803"); - HiveDecimal dec2 = HiveDecimal.create("3.0000123456789"); - HiveDecimal result = dec1.multiply(dec2); - Assert.assertTrue(result == null); - - dec1 = HiveDecimal.create("178613588865784752580323232232323444.4"); - dec2 = HiveDecimal.create("178613588865784752580302323232.3"); - Assert.assertNull(dec1.multiply(dec2)); // i.e. Overflow. - - dec1 = HiveDecimal.create("47.324"); - dec2 = HiveDecimal.create("9232.309"); - Assert.assertEquals("436909.791116", dec1.multiply(dec2).toString()); - - dec1 = HiveDecimal.create("3.140"); - dec2 = HiveDecimal.create("1.00"); - Assert.assertEquals("3.14", dec1.multiply(dec2).toString()); - - dec1 = HiveDecimal.create("43.010"); - dec2 = HiveDecimal.create("2"); - Assert.assertEquals("86.02", dec1.multiply(dec2).toString()); - } - - @Test - public void testMultiply2() { - // 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD - HiveDecimal dec1 = HiveDecimal.create("0.09765625"); - HiveDecimal dec2 = HiveDecimal.create("0.09765625"); - HiveDecimal dec3 = HiveDecimal.create("0.0125"); - HiveDecimal dec4 = HiveDecimal.create("578992"); - HiveDecimal result1 = dec1.multiply(dec2); - Assert.assertNotNull(result1); - HiveDecimal result2 = result1.multiply(dec3); - Assert.assertNotNull(result2); - HiveDecimal result = result2.multiply(dec4); - Assert.assertNotNull(result); - Assert.assertEquals("69.0212249755859375", result.toString()); - } - - @Test - public void testPow() { - - HiveDecimal dec; - - dec = HiveDecimal.create("3.00001415926"); - HiveDecimal decPow2 = dec.pow(2); - HiveDecimal decMultiplyTwice = dec.multiply(dec); - Assert.assertEquals(decPow2, decMultiplyTwice); - - dec = HiveDecimal.create("0.000017861358882"); - dec = dec.pow(3); - Assert.assertNull(dec); - - dec = HiveDecimal.create("3.140"); - Assert.assertEquals("9.8596", dec.pow(2).toString()); - } - - @Test - public void testScaleByPowerOfTen() { - - HiveDecimalV1 oldDec; - HiveDecimal dec; - HiveDecimalV1 oldResultDec; - HiveDecimal resultDec; - - //********************************************************************************************** - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create( - "1"); - Assert.assertEquals(0, oldDec.scale()); - oldResultDec = oldDec.scaleByPowerOfTen(2); - Assert.assertEquals( - "100", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create( - "1"); - Assert.assertEquals(0, dec.scale()); - // resultDec = dec.scaleByPowerOfTen(2); - // Assert.assertEquals( - // "100", resultDec.toString()); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create( - "0.00000000000000000000000000000000000001"); - Assert.assertEquals(38, oldDec.scale()); - oldResultDec = oldDec.scaleByPowerOfTen(2); - Assert.assertEquals( - "0.000000000000000000000000000000000001", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create( - "0.00000000000000000000000000000000000001"); - Assert.assertEquals(38, dec.scale()); - resultDec = dec.scaleByPowerOfTen(2); - Assert.assertEquals( - "0.000000000000000000000000000000000001", resultDec.toString()); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create( - "0.00000000000000000000000000000000000001"); - Assert.assertEquals(38, oldDec.scale()); - oldResultDec = oldDec.scaleByPowerOfTen(38); - Assert.assertEquals( - "1", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create( - "0.00000000000000000000000000000000000001"); - Assert.assertEquals(38, dec.scale()); - resultDec = dec.scaleByPowerOfTen(38); - Assert.assertEquals( - "1", resultDec.toString()); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create( - "0.00000000000000000000000000000000000001"); - Assert.assertEquals(38, oldDec.scale()); - oldResultDec = oldDec.scaleByPowerOfTen(2 * 38 - 1); - Assert.assertEquals( - "10000000000000000000000000000000000000", oldResultDec.toString()); - Assert.assertEquals(0, oldResultDec.scale()); - //--------------------------------------------------- - dec = HiveDecimal.create( - "0.00000000000000000000000000000000000001"); - Assert.assertEquals(38, dec.scale()); - resultDec = dec.scaleByPowerOfTen(2 * 38 - 1); - Assert.assertEquals( - "10000000000000000000000000000000000000", resultDec.toString()); - Assert.assertEquals(0, resultDec.scale()); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create( - "0.00000000000000000000000000000000000001"); - Assert.assertEquals(38, oldDec.scale()); - oldResultDec = oldDec.scaleByPowerOfTen(2 * 38); - Assert.assertTrue(oldResultDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create( - "0.00000000000000000000000000000000000001"); - Assert.assertEquals(38, dec.scale()); - resultDec = dec.scaleByPowerOfTen(2 * 38); - Assert.assertTrue(resultDec == null); - - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create( - "0.00000000000000000000000000000000000022"); - Assert.assertEquals(38, oldDec.scale()); - oldResultDec = oldDec.scaleByPowerOfTen(38); - Assert.assertEquals( - "22", oldResultDec.toString()); - Assert.assertEquals(0, oldResultDec.scale()); - //--------------------------------------------------- - dec = HiveDecimal.create( - "0.00000000000000000000000000000000000022"); - Assert.assertEquals(38, dec.scale()); - resultDec = dec.scaleByPowerOfTen(38); - Assert.assertEquals( - "22", resultDec.toString()); - Assert.assertEquals(0, resultDec.scale()); - - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("3.00001415926"); - Assert.assertEquals(11, oldDec.scale()); - oldResultDec = oldDec.scaleByPowerOfTen(2); - Assert.assertEquals("300.001415926", oldResultDec.toString()); - Assert.assertEquals(9, oldResultDec.scale()); - oldResultDec = oldDec.scaleByPowerOfTen(5); - Assert.assertEquals("300001.415926", oldResultDec.toString()); - Assert.assertEquals(6, oldResultDec.scale()); - oldResultDec = oldDec.scaleByPowerOfTen(18); - Assert.assertEquals("3000014159260000000", oldResultDec.toString()); - Assert.assertEquals(0, oldResultDec.scale()); - oldResultDec = oldDec.scaleByPowerOfTen(35); - Assert.assertEquals("300001415926000000000000000000000000", oldResultDec.toString()); - Assert.assertEquals(0, oldResultDec.scale()); - oldResultDec = oldDec.scaleByPowerOfTen(37); - Assert.assertEquals("30000141592600000000000000000000000000", oldResultDec.toString()); - Assert.assertEquals(0, oldResultDec.scale()); - //--------------------------------------------------- - dec = HiveDecimal.create("3.00001415926"); - Assert.assertEquals(11, dec.scale()); - Assert.assertEquals(1, dec.integerDigitCount()); - resultDec = dec.scaleByPowerOfTen(2); - Assert.assertEquals("300.001415926", resultDec.toString()); - Assert.assertEquals(9, resultDec.scale()); - Assert.assertEquals(3, resultDec.integerDigitCount()); - resultDec = dec.scaleByPowerOfTen(5); - Assert.assertEquals("300001.415926", resultDec.toString()); - Assert.assertEquals(6, resultDec.scale()); - Assert.assertEquals(6, resultDec.integerDigitCount()); - resultDec = dec.scaleByPowerOfTen(18); - Assert.assertEquals("3000014159260000000", resultDec.toString()); - Assert.assertEquals(0, resultDec.scale()); - Assert.assertEquals(19, resultDec.integerDigitCount()); - resultDec = dec.scaleByPowerOfTen(35); - Assert.assertEquals("300001415926000000000000000000000000", resultDec.toString()); - Assert.assertEquals(0, resultDec.scale()); - Assert.assertEquals(36, resultDec.integerDigitCount()); - resultDec = dec.scaleByPowerOfTen(37); - Assert.assertEquals("30000141592600000000000000000000000000", resultDec.toString()); - Assert.assertEquals(0, resultDec.scale()); - Assert.assertEquals(38, resultDec.integerDigitCount()); - } - - @Test - public void testSingleWordDivision() { - - HiveDecimalV1 oldDec1; - HiveDecimalV1 oldDec2; - HiveDecimalV1 oldResultDec; - - HiveDecimal dec1; - HiveDecimal dec2; - HiveDecimal resultDec; - - //--------------------------------------------------- - oldDec1 = HiveDecimalV1.create("839293"); - oldDec2 = HiveDecimalV1.create("8"); - oldResultDec = oldDec1.divide(oldDec2); - Assert.assertEquals("104911.625", oldResultDec.toString()); - //--------------------------------------------------- - dec1 = HiveDecimal.create("839293"); - dec2 = HiveDecimal.create("8"); - resultDec = dec1.divide(dec2); - Assert.assertEquals("104911.625", resultDec.toString()); // UNDONE - - //--------------------------------------------------- - oldDec1 = HiveDecimalV1.create("1"); - oldDec2 = HiveDecimalV1.create("3"); - oldResultDec = oldDec1.divide(oldDec2); - Assert.assertEquals("0.33333333333333333333333333333333333333", oldResultDec.toString()); - //--------------------------------------------------- - dec1 = HiveDecimal.create("1"); - dec2 = HiveDecimal.create("3"); - resultDec = dec1.divide(dec2); - Assert.assertEquals("0.33333333333333333333333333333333333333", resultDec.toString()); // UNDONE - - //--------------------------------------------------- - oldDec1 = HiveDecimalV1.create("1"); - oldDec2 = HiveDecimalV1.create("9"); - oldResultDec = oldDec1.divide(oldDec2); - Assert.assertEquals("0.11111111111111111111111111111111111111", oldResultDec.toString()); - //--------------------------------------------------- - dec1 = HiveDecimal.create("1"); - dec2 = HiveDecimal.create("9"); - resultDec = dec1.divide(dec2); - Assert.assertEquals("0.11111111111111111111111111111111111111", resultDec.toString()); // UNDONE - - //--------------------------------------------------- - oldDec1 = HiveDecimalV1.create("22"); - oldDec2 = HiveDecimalV1.create("7"); - oldResultDec = oldDec1.divide(oldDec2); - Assert.assertEquals("3.1428571428571428571428571428571428571", oldResultDec.toString()); - //--------------------------------------------------- - dec1 = HiveDecimal.create("22"); - dec2 = HiveDecimal.create("7"); - resultDec = dec1.divide(dec2); - Assert.assertEquals("3.1428571428571428571428571428571428571", resultDec.toString()); // UNDONE - - //--------------------------------------------------- - oldDec1 = HiveDecimalV1.create("1"); - oldDec2 = HiveDecimalV1.create("81"); - oldResultDec = oldDec1.divide(oldDec2); - Assert.assertEquals("0.01234567901234567901234567901234567901", oldResultDec.toString()); - //--------------------------------------------------- - dec1 = HiveDecimal.create("1"); - dec2 = HiveDecimal.create("81"); - resultDec = dec1.divide(dec2); - Assert.assertEquals("0.01234567901234567901234567901234567901", resultDec.toString()); // UNDONE - - //--------------------------------------------------- - oldDec1 = HiveDecimalV1.create("425"); - oldDec2 = HiveDecimalV1.create("1000000000000000"); - oldResultDec = oldDec1.divide(oldDec2); - Assert.assertEquals("0.000000000000425", oldResultDec.toString()); - //--------------------------------------------------- - dec1 = HiveDecimal.create("425"); - dec2 = HiveDecimal.create("1000000000000000"); - resultDec = dec1.divide(dec2); - Assert.assertEquals("0.000000000000425", resultDec.toString()); // UNDONE - - //--------------------------------------------------- - oldDec1 = HiveDecimalV1.create("0.000000000088"); - oldDec2 = HiveDecimalV1.create("1000000000000000"); - oldResultDec = oldDec1.divide(oldDec2); - Assert.assertEquals("0.000000000000000000000000088", oldResultDec.toString()); - Assert.assertEquals(27, oldResultDec.scale()); - //--------------------------------------------------- - dec1 = HiveDecimal.create("0.000000000088"); - dec2 = HiveDecimal.create("1000000000000000"); - resultDec = dec1.divide(dec2); - Assert.assertEquals("0.000000000000000000000000088", resultDec.toString()); // UNDONE - Assert.assertEquals(27, resultDec.scale()); - } - - @Test - public void testDivide() { - HiveDecimal dec1 = HiveDecimal.create("3.14"); - HiveDecimal dec2 = HiveDecimal.create("3"); - Assert.assertNotNull(dec1.divide(dec2)); - - dec1 = HiveDecimal.create("15"); - dec2 = HiveDecimal.create("5"); - Assert.assertEquals("3", dec1.divide(dec2).toString()); - - dec1 = HiveDecimal.create("3.140"); - dec2 = HiveDecimal.create("1.00"); - Assert.assertEquals("3.14", dec1.divide(dec2).toString()); - } - - @Test - public void testPlus() { - - HiveDecimalV1 oldDec; - HiveDecimalV1 oldDec2; - HiveDecimalV1 oldResultDec; - - HiveDecimal dec; - HiveDecimal dec1; - HiveDecimal dec2; - HiveDecimal resultDec; - - String decStr; - String decStr2; - - dec1 = HiveDecimal.create("3.140"); - dec1.validate(); - dec2 = HiveDecimal.create("1.00"); - dec2.validate(); - resultDec = dec1.add(dec2); - resultDec.validate(); - Assert.assertEquals("4.14", resultDec.toString()); - - decStr = "3.140"; - decStr2 = "1.00"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.add(oldDec2); - Assert.assertEquals("4.14", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.add(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("4.14", resultDec.toString()); - Assert.assertEquals(1, resultDec.integerDigitCount()); - - decStr = "3.140"; - decStr2 = "1.00000008733"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.add(oldDec2); - Assert.assertEquals("4.14000008733", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.add(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("4.14000008733", resultDec.toString()); - Assert.assertEquals(1, resultDec.integerDigitCount()); - - decStr = "3.140"; - decStr2 = "1.00000000000000000008733"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.add(oldDec2); - Assert.assertEquals("4.14000000000000000008733", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.add(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("4.14000000000000000008733", resultDec.toString()); - Assert.assertEquals(1, resultDec.integerDigitCount()); - - decStr = "30000000000.140"; - decStr2 = "1.00000000000000000008733"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.add(oldDec2); - Assert.assertEquals("30000000001.14000000000000000008733", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.add(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("30000000001.14000000000000000008733", resultDec.toString()); - // Assert.assertEquals(1, resultDec.integerDigitCount()); - - decStr = "300000000000000.140"; - decStr2 = "1.00000000000000000008733"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.add(oldDec2); - Assert.assertEquals("300000000000001.14000000000000000008733", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.add(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("300000000000001.14000000000000000008733", resultDec.toString()); - // Assert.assertEquals(1, resultDec.integerDigitCount()); - - // Edge case? - decStr = "3000000000000000.140"; - decStr2 = "1.00000000000000000008733"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.add(oldDec2); - Assert.assertEquals("3000000000000001.1400000000000000000873", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.add(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("3000000000000001.1400000000000000000873", resultDec.toString()); - // Assert.assertEquals(1, resultDec.integerDigitCount()); - - decStr = "300000000000000000000000.14"; - decStr2 = "0.0000055555555550008733"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.add(oldDec2); - Assert.assertEquals("300000000000000000000000.14000555555556", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.add(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("300000000000000000000000.14000555555556", resultDec.toString()); - // Assert.assertEquals(1, resultDec.integerDigitCount()); - - decStr = "300000000000000000000000.14"; - decStr2 = "0.000005555555555000873355"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.add(oldDec2); - Assert.assertEquals("300000000000000000000000.14000555555556", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.add(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("300000000000000000000000.14000555555556", resultDec.toString()); - // Assert.assertEquals(1, resultDec.integerDigitCount()); - - - - // Example from HiveDecimal.add header comments. - decStr = "598575157855521918987423259.94094"; - decStr2 = "0.0000000000006711991169422033"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.add(oldDec2); - Assert.assertEquals("598575157855521918987423259.94094", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.add(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("598575157855521918987423259.94094", resultDec.toString()); - Assert.assertEquals(27, resultDec.integerDigitCount()); - - decStr = "598575157855521918987423259.94094"; - decStr2 = "0.5555555555556711991169422033"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.add(oldDec2); - Assert.assertEquals("598575157855521918987423260.49649555556", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.add(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("598575157855521918987423260.49649555556", resultDec.toString()); - Assert.assertEquals(27, resultDec.integerDigitCount()); - - decStr = "199999999.99995"; - decStr2 = "100000000.00005"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.add(oldDec2); - Assert.assertEquals("300000000", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.add(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("300000000", resultDec.toString()); - Assert.assertEquals(9, resultDec.integerDigitCount()); - - dec1 = HiveDecimal.create("99999999999999999999999999999999999999"); - dec1.validate(); - Assert.assertEquals(38, dec1.integerDigitCount()); - dec2 = HiveDecimal.create("1"); - dec2.validate(); - Assert.assertNull(dec1.add(dec2)); - } - - @Test - public void testAdd() { - - HiveDecimalV1 oldDec; - HiveDecimalV1 oldDec2; - HiveDecimalV1 oldResultDec; - - HiveDecimal dec; - HiveDecimal dec2; - HiveDecimal resultDec; - - // Use the example from HIVE-13423 where the integer digits of the result exceed the - // enforced precision/scale. - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("98765432109876543210.12345"); - oldResultDec = oldDec.add(oldDec); - Assert.assertEquals("197530864219753086420.2469", oldResultDec.toString()); - oldResultDec = HiveDecimalV1.enforcePrecisionScale(oldResultDec, 38, 18); - Assert.assertTrue(oldResultDec == null); - //--------------------------------------------------- - dec = HiveDecimal.create("98765432109876543210.12345"); - assertTrue(dec != null); - dec.validate(); - resultDec = dec.add(dec); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("197530864219753086420.2469", resultDec.toString()); - // Assert.assertEquals(21, resultDec.integerDigitCount()); - resultDec = HiveDecimal.enforcePrecisionScale(resultDec, 38, 18); - Assert.assertTrue(resultDec == null); - - // Make sure zero trimming doesn't extend into the integer digits. - //--------------------------------------------------- - oldDec = HiveDecimalV1.create("199999999.99995"); - oldDec2 = HiveDecimalV1.create("100000000.00005"); - oldResultDec = oldDec.add(oldDec2); - Assert.assertEquals("300000000", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create("199999999.99995"); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create("100000000.00005"); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.add(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("300000000", resultDec.toString()); - Assert.assertEquals(9, resultDec.integerDigitCount()); - } - - @Test - public void testMinus() { - - HiveDecimalV1 oldDec; - HiveDecimalV1 oldDec2; - HiveDecimalV1 oldResultDec; - - HiveDecimal dec; - HiveDecimal dec1; - HiveDecimal dec2; - HiveDecimal resultDec; - - String decStr; - String decStr2; - - dec1 = HiveDecimal.create("3.140"); - dec1.validate(); - dec2 = HiveDecimal.create("1.00"); - dec2.validate(); - resultDec = dec1.add(dec2); - resultDec.validate(); - Assert.assertEquals("4.14", resultDec.toString()); - - decStr = "3.140"; - decStr2 = "1.00"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.subtract(oldDec2); - Assert.assertEquals("2.14", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.subtract(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("2.14", resultDec.toString()); - // Assert.assertEquals(1, resultDec.integerDigitCount()); - - decStr = "3.140"; - decStr2 = "1.00000008733"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.subtract(oldDec2); - Assert.assertEquals("2.13999991267", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.subtract(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("2.13999991267", resultDec.toString()); - // Assert.assertEquals(1, resultDec.integerDigitCount()); - - decStr = "3.140"; - decStr2 = "1.00000000000000000008733"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.subtract(oldDec2); - Assert.assertEquals("2.13999999999999999991267", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.subtract(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("2.13999999999999999991267", resultDec.toString()); - // Assert.assertEquals(1, resultDec.integerDigitCount()); - - decStr = "30000000000.140"; - decStr2 = "1.00000000000000000008733"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.subtract(oldDec2); - Assert.assertEquals("29999999999.13999999999999999991267", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.subtract(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("29999999999.13999999999999999991267", resultDec.toString()); - // Assert.assertEquals(1, resultDec.integerDigitCount()); - - decStr = "300000000000000.140"; - decStr2 = "1.00000000000000000008733"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.subtract(oldDec2); - Assert.assertEquals("299999999999999.13999999999999999991267", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.subtract(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("299999999999999.13999999999999999991267", resultDec.toString()); - // Assert.assertEquals(1, resultDec.integerDigitCount()); - - // Edge case? - decStr = "3000000000000000.140"; - decStr2 = "1.00000000000000000008733"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.subtract(oldDec2); - Assert.assertEquals("2999999999999999.1399999999999999999127", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.subtract(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("2999999999999999.1399999999999999999127", resultDec.toString()); - // Assert.assertEquals(1, resultDec.integerDigitCount()); - - decStr = "300000000000000000000000.14"; - decStr2 = "0.0000055555555550008733"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.subtract(oldDec2); - Assert.assertEquals("300000000000000000000000.13999444444444", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.subtract(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("300000000000000000000000.13999444444444", resultDec.toString()); - // Assert.assertEquals(1, resultDec.integerDigitCount()); - - decStr = "300000000000000000000000.14"; - decStr2 = "0.000005555555555000873355"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.subtract(oldDec2); - Assert.assertEquals("300000000000000000000000.13999444444444", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.subtract(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("300000000000000000000000.13999444444444", resultDec.toString()); - // Assert.assertEquals(1, resultDec.integerDigitCount()); - - // Example from HiveDecimal.subtract header comments. - decStr = "598575157855521918987423259.94094"; - decStr2 = "0.0000000000006711991169422033"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.subtract(oldDec2); - Assert.assertEquals("598575157855521918987423259.94094", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.subtract(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("598575157855521918987423259.94094", resultDec.toString()); - // Assert.assertEquals(1, resultDec.integerDigitCount()); - - decStr = "598575157855521918987423259.94094"; - decStr2 = "0.5555555555556711991169422033"; - //--------------------------------------------------- - oldDec = HiveDecimalV1.create(decStr); - oldDec2 = HiveDecimalV1.create(decStr2); - oldResultDec = oldDec.subtract(oldDec2); - Assert.assertEquals("598575157855521918987423259.38538444444", oldResultDec.toString()); - //--------------------------------------------------- - dec = HiveDecimal.create(decStr); - assertTrue(dec != null); - dec.validate(); - dec2 = HiveDecimal.create(decStr2); - assertTrue(dec2 != null); - dec2.validate(); - resultDec = dec.subtract(dec2); - assertTrue(resultDec != null); - resultDec.validate(); - Assert.assertEquals("598575157855521918987423259.38538444444", resultDec.toString()); - // Assert.assertEquals(1, resultDec.integerDigitCount()); - } - - @Test - public void testSubtract() { - HiveDecimal dec1 = HiveDecimal.create("3.140"); - assertTrue(dec1 != null); - dec1.validate(); - HiveDecimal dec2 = HiveDecimal.create("1.00"); - assertTrue(dec2 != null); - dec2.validate(); - HiveDecimal result = dec1.subtract(dec2); - assertTrue(result != null); - result.validate(); - Assert.assertEquals("2.14", result.toString()); - - dec1 = HiveDecimal.create("0.00001786135888657847525803"); - assertTrue(dec1 != null); - dec1.validate(); - dec2 = HiveDecimal.create("3.0000123456789"); - assertTrue(dec2 != null); - dec2.validate(); - result = dec1.subtract(dec2); - assertTrue(result != null); - result.validate(); - Assert.assertEquals("-2.99999448432001342152474197", result.toString()); - } - - @Test - public void testPosMod() { - HiveDecimal hd1 = HiveDecimal.create("-100.91"); - assertTrue(hd1 != null); - hd1.validate(); - HiveDecimal hd2 = HiveDecimal.create("9.8"); - assertTrue(hd2 != null); - hd2.validate(); - HiveDecimal dec = hd1.remainder(hd2).add(hd2).remainder(hd2); - assertTrue(dec != null); - dec.validate(); - Assert.assertEquals("6.89", dec.toString()); - } - - @Test - public void testHashCode() { - Assert.assertEquals(HiveDecimal.create("9").hashCode(), HiveDecimal.create("9.00").hashCode()); - Assert.assertEquals(HiveDecimal.create("0").hashCode(), HiveDecimal.create("0.00").hashCode()); - } - - @Test - public void testException() { - HiveDecimal dec = HiveDecimal.create("3.1415.926"); - Assert.assertNull(dec); - dec = HiveDecimal.create("3abc43"); - Assert.assertNull(dec); - } - - @Test - public void testBinaryConversion() { - Random r = new Random(2399); - for (String decString : specialDecimalStrings) { - doTestBinaryConversion(decString, r); - } - } - - private void doTestBinaryConversion(String num, Random r) { - int scale = r.nextInt(HiveDecimal.MAX_SCALE); - HiveDecimal dec = HiveDecimal.create(num); - if (dec == null) { - return; - } - byte[] d = dec.setScale(scale).unscaledValue().toByteArray(); - HiveDecimal roundedDec = dec.setScale(scale, HiveDecimal.ROUND_HALF_UP); - Assert.assertEquals(roundedDec, HiveDecimal.create(new BigInteger(d), scale)); - } - -//------------------------------------------------------------------------------------------------ - - @Test - public void testDecimalsWithOneOne() { - doTestDecimalsWithPrecisionScale(decimal_1_1_txt, 1, 1); - } - - @Test - public void testDecimalsWithKv7Keys() { - doTestDecimalsWithPrecisionScale(kv7_txt_keys, 38, 18); - } - - public void doTestDecimalsWithPrecisionScale(String[] decStrings, int precision, int scale) { - - HiveDecimalV1 oldSum = HiveDecimalV1.create(0); - HiveDecimalWritable sum = new HiveDecimalWritable(0); - - for (int i = 0; i < decStrings.length; i++) { - - String string = decStrings[i]; - - HiveDecimalV1 oldDec = HiveDecimalV1.create(string); - - HiveDecimalV1 resultOldDec; - if (oldDec == null) { - resultOldDec = null; - } else { - resultOldDec = HiveDecimalV1.enforcePrecisionScale(oldDec, precision, scale); - } - - HiveDecimal dec = HiveDecimal.create(string); - - if (oldDec == null) { - Assert.assertTrue(dec == null); - continue; - } - HiveDecimal resultDec = HiveDecimal.enforcePrecisionScale(dec, precision, scale); - if (resultOldDec == null) { - Assert.assertTrue(resultDec == null); - continue; - } - - Assert.assertEquals(resultOldDec.toString(), resultDec.toString()); - Assert.assertEquals(resultOldDec.toFormatString(scale), resultDec.toFormatString(scale)); - - oldSum = oldSum.add(resultOldDec); - sum.mutateAdd(resultDec); - } - - Assert.assertEquals(oldSum.toString(), sum.toString()); - } - -//------------------------------------------------------------------------------------------------ - - @Test - public void testDecimalsWithOneOneWritable() { - doTestDecimalsWithPrecisionScaleWritable(decimal_1_1_txt, 1, 1); - } - - @Test - public void testDecimalsWithKv7KeysWritable() { - doTestDecimalsWithPrecisionScaleWritable(kv7_txt_keys, 38, 18); - } - - public void doTestDecimalsWithPrecisionScaleWritable(String[] decStrings, int precision, int scale) { - - HiveDecimalV1 oldSum = HiveDecimalV1.create(0); - HiveDecimalWritable sum = new HiveDecimalWritable(0); - - for (int i = 0; i < decStrings.length; i++) { - String string = decStrings[i]; - - HiveDecimalV1 oldDec = HiveDecimalV1.create(string); - HiveDecimalV1 resultOldDec; - if (oldDec == null) { - resultOldDec = null; - } else { - resultOldDec = HiveDecimalV1.enforcePrecisionScale(oldDec, precision, scale); - } - - HiveDecimalWritable decWritable = new HiveDecimalWritable(string); - if (oldDec == null) { - Assert.assertTrue(!decWritable.isSet()); - continue; - } - decWritable.mutateEnforcePrecisionScale(precision, scale);; - if (resultOldDec == null) { - Assert.assertTrue(!decWritable.isSet()); - continue; - } - - Assert.assertEquals(resultOldDec.toString(), decWritable.toString()); - Assert.assertEquals(resultOldDec.toFormatString(scale), decWritable.toFormatString(scale)); - - oldSum = oldSum.add(resultOldDec); - sum.mutateAdd(decWritable); - } - - Assert.assertEquals(oldSum.toString(), sum.toString()); - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testSort() { - doTestSort(decimal_1_1_txt); - } - - @Test - public void testSortSpecial() { - doTestSort(specialDecimalStrings); - } - - @Test - public void testSortRandom() { - Random r = new Random(14434); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestSortRandom(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestSortRandom(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - public void doTestSortRandom(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - String[] randomStrings = new String[POUND_FACTOR]; - - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - randomStrings[i] = bigDecimal.toString(); - } - - doTestSort(randomStrings); - } - - public void doTestSort(String[] decStrings) { - - HiveDecimalV1[] oldDecSortArray = new HiveDecimalV1[decStrings.length]; - HiveDecimal[] decSortArray = new HiveDecimal[decStrings.length]; - - int count = 0; - for (int i = 0; i < decStrings.length; i++) { - String string = decStrings[i]; - - HiveDecimalV1 oldDec = HiveDecimalV1.create(string); - if (oldDec == null) { - continue; - } - if (isTenPowerBug(oldDec.toString())) { - continue; - } - oldDecSortArray[count] = oldDec; - - HiveDecimal dec = HiveDecimal.create(string); - if (dec == null) { - Assert.fail(); - } - decSortArray[count] = dec; - count++; - } - - oldDecSortArray = Arrays.copyOf(oldDecSortArray, count); - decSortArray = Arrays.copyOf(decSortArray, count); - - Arrays.sort(oldDecSortArray); - Arrays.sort(decSortArray); - - for (int i = 0; i < count; i++) { - String oldDecString = oldDecSortArray[i].toString(); - String decString = decSortArray[i].toString(); - - if (!oldDecString.equals(decString)) { - Assert.fail(); - } - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomCreateFromBigDecimal() { - Random r = new Random(14434); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomCreateFromBigDecimal(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomCreateFromBigDecimal(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomCreateFromBigDecimal(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestCreateFromBigDecimal(bigDecimal); - } - } - - @Test - public void testCreateFromBigDecimalSpecial() { - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestCreateFromBigDecimal(bigDecimal); - } - } - - private void doTestCreateFromBigDecimal(BigDecimal bigDecimal) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec = HiveDecimal.create(bigDecimal); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - Assert.assertEquals(oldDec.toString(), dec.toString()); - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomCreateFromBigDecimalNoRound() { - Random r = new Random(14434); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomCreateFromBigDecimalNoRound(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomCreateFromBigDecimalNoRound(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomCreateFromBigDecimalNoRound(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestCreateFromBigDecimalNoRound(bigDecimal); - } - } - - @Test - public void testCreateFromBigDecimalNoRoundSpecial() { - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestCreateFromBigDecimalNoRound(bigDecimal); - } - } - - private void doTestCreateFromBigDecimalNoRound(BigDecimal bigDecimal) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal, /* allowRounding */ false); - HiveDecimal dec = HiveDecimal.create(bigDecimal, /* allowRounding */ false); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - if (dec == null) { - Assert.fail(); - } - dec.validate(); - - Assert.assertEquals(oldDec.toString(), dec.toString()); - - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testCreateFromBigDecimalNegativeScaleSpecial() { - Random r = new Random(223965); - for (BigDecimal bigDecimal : specialBigDecimals) { - int negativeScale = -(0 + r.nextInt(38 + 1)); - bigDecimal = bigDecimal.setScale(negativeScale, BigDecimal.ROUND_HALF_UP); - doTestCreateFromBigDecimalNegativeScale(bigDecimal); - } - } - - private void doTestCreateFromBigDecimalNegativeScale(BigDecimal bigDecimal) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - HiveDecimal dec = HiveDecimal.create(bigDecimal); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - Assert.assertEquals(oldDec.toString(), dec.toString()); - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomCreateFromBigInteger() { - doTestRandomCreateFromBigInteger(standardAlphabet); - } - - @Test - public void testRandomCreateFromBigIntegerSparse() { - for (String digitAlphabet : sparseAlphabets) { - doTestRandomCreateFromBigInteger(digitAlphabet); - } - } - - private void doTestRandomCreateFromBigInteger(String digitAlphabet) { - - Random r = new Random(11241); - for (int i = 0; i < POUND_FACTOR; i++) { - BigInteger bigInteger = randHiveBigInteger(r, digitAlphabet); - - doTestCreateFromBigInteger(bigInteger); - } - } - - @Test - public void testCreateFromBigIntegerSpecial() { - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestCreateFromBigInteger(bigDecimal.unscaledValue()); - } - } - - private void doTestCreateFromBigInteger(BigInteger bigInteger) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigInteger); - HiveDecimal dec = HiveDecimal.create(bigInteger); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - Assert.assertEquals(oldDec.toString(), dec.toString()); - - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomCreateFromBigIntegerScale() { - doTestRandomCreateFromBigIntegerScale(standardAlphabet, false); - } - - @Test - public void testRandomCreateFromBigIntegerScaleFractionsOnly() { - doTestRandomCreateFromBigIntegerScale(standardAlphabet, true); - } - - @Test - public void testRandomCreateFromBigIntegerScaleSparse() { - for (String digitAlphabet : sparseAlphabets) { - doTestRandomCreateFromBigIntegerScale(digitAlphabet, false); - } - } - - private void doTestRandomCreateFromBigIntegerScale(String digitAlphabet, boolean fractionsOnly) { - - Random r = new Random(4448); - for (int i = 0; i < POUND_FACTOR; i++) { - BigInteger bigInteger = randHiveBigInteger(r, digitAlphabet); - - int scale; - if (fractionsOnly) { - scale = 1 + r.nextInt(38); - } else { - scale = 0 + r.nextInt(38 + 1); - } - - doTestCreateFromBigIntegerScale(bigInteger, scale); - } - } - - @Test - public void testCreateFromBigIntegerScaleSpecial() { - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestCreateFromBigIntegerScale(bigDecimal.unscaledValue(), bigDecimal.scale()); - } - } - - private void doTestCreateFromBigIntegerScale(BigInteger bigInteger, int scale) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigInteger, scale); - HiveDecimal dec = HiveDecimal.create(bigInteger, scale); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - Assert.assertEquals(oldDec.toString(), dec.toString()); - - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomSetFromDouble() { - Random r = new Random(14434); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomSetFromDouble(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomSetFromDouble(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomSetFromDouble(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestSetFromDouble(bigDecimal.doubleValue()); - } - } - - private void doTestRandomSetFromDouble() { - - Random r = new Random(94762); - for (int i = 0; i < POUND_FACTOR; i++) { - double randomDouble = r.nextDouble(); - - doTestSetFromDouble(randomDouble); - } - } - - @Test - public void testSetFromDoubleSpecial() { - - for (String specialString : specialDecimalStrings) { - double specialDouble = Double.valueOf(specialString); - doTestSetFromDouble(specialDouble); - } - } - - private void doTestSetFromDouble(double doubleValue) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(Double.toString(doubleValue)); - if (oldDec == null) { - return; - } - HiveDecimal dec = HiveDecimal.create(doubleValue); - if (dec == null) { - Assert.fail(); - } - dec.validate(); - if (!oldDec.toString().equals(dec.toString())) { - BigDecimal bigDecimal = new BigDecimal(dec.toString()); - for (int i = 16; i < 18;i++) { - BigDecimal trial = bigDecimal.setScale(i, HiveDecimal.ROUND_HALF_UP); - } - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomCreateFromString() { - Random r = new Random(1221); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomCreateFromString(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomCreateFromString(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomCreateFromString(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestCreateFromString(bigDecimal); - } - } - - @Test - public void testCreateFromStringSpecial() { - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestCreateFromString(bigDecimal); - } - } - - private void doTestCreateFromString(BigDecimal bigDecimal) { - - String decString = bigDecimal.toPlainString(); - - HiveDecimalV1 oldDec = HiveDecimalV1.create(decString); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec = HiveDecimal.create(decString); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - Assert.assertEquals(oldDec.toString(), dec.toString()); - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomCreateFromStringPadded() { - Random r = new Random(9774); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomCreateFromStringPadded(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomCreateFromStringPadded(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomCreateFromStringPadded(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestCreateFromStringPadded(bigDecimal); - } - } - - @Test - public void testCreateFromStringPaddedSpecial() { - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestCreateFromStringPadded(bigDecimal); - } - } - - private void doTestCreateFromStringPadded(BigDecimal bigDecimal) { - - String decString = bigDecimal.toPlainString(); - String decString1 = " " + decString; - String decString2 = decString + " "; - String decString3 = " " + decString + " "; - String decString4 = " " + decString; - String decString5 = decString + " "; - String decString6 = " " + decString + " "; - - HiveDecimalV1 oldDec; - HiveDecimal dec; - - oldDec = HiveDecimalV1.create(decString); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - - dec = HiveDecimal.create(decString1, true); - if (oldDec == null) { - assertTrue(dec == null); - } else { - assertTrue(dec != null); - dec.validate(); - - Assert.assertEquals(oldDec.toString(), dec.toString()); - } - - dec = HiveDecimal.create(decString2, true); - if (oldDec == null) { - assertTrue(dec == null); - } else { - assertTrue(dec != null); - dec.validate(); - - Assert.assertEquals(oldDec.toString(), dec.toString()); - } - - dec = HiveDecimal.create(decString3, true); - if (oldDec == null) { - assertTrue(dec == null); - } else { - assertTrue(dec != null); - dec.validate(); - - Assert.assertEquals(oldDec.toString(), dec.toString()); - } - - dec = HiveDecimal.create(decString4, true); - if (oldDec == null) { - assertTrue(dec == null); - } else { - assertTrue(dec != null); - dec.validate(); - - Assert.assertEquals(oldDec.toString(), dec.toString()); - } - - dec = HiveDecimal.create(decString5, true); - if (oldDec == null) { - assertTrue(dec == null); - } else { - assertTrue(dec != null); - dec.validate(); - - Assert.assertEquals(oldDec.toString(), dec.toString()); - } - - dec = HiveDecimal.create(decString6, true); - if (oldDec == null) { - assertTrue(dec == null); - } else { - assertTrue(dec != null); - dec.validate(); - - Assert.assertEquals(oldDec.toString(), dec.toString()); - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomCreateFromStringExponent() { - Random r = new Random(297111); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomCreateFromStringPadded(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomCreateFromStringPadded(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomCreateFromStringExponent(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestCreateFromStringExponent(bigDecimal); - } - } - - @Test - public void testCreateFromStringExponentSpecial() { - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestCreateFromStringExponent(bigDecimal); - } - } - - private void doTestCreateFromStringExponent(BigDecimal bigDecimal) { - - // Use toString which will have exponents instead of toPlainString. - String decString = bigDecimal.toString(); - - HiveDecimalV1 oldDec = HiveDecimalV1.create(decString); - HiveDecimal dec = HiveDecimal.create(decString); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - Assert.assertEquals(oldDec.toString(), dec.toString()); - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomLongValue() { - Random r = new Random(73293); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomLongValue(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomLongValue(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomLongValue(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestLongValue(bigDecimal); - } - } - - @Test - public void testLongValueSpecial() { - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestLongValue(bigDecimal); - } - } - - private void doTestLongValue(BigDecimal bigDecimal) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec = HiveDecimal.create(bigDecimal); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - BigDecimal bigDecimalOldDec = oldDec.bigDecimalValue(); - BigDecimal bigDecimalDec = dec.bigDecimalValue(); - Assert.assertEquals(bigDecimalOldDec, bigDecimalDec); - - BigDecimal bigDecimalFloor = bigDecimalDec.setScale(0, BigDecimal.ROUND_DOWN); - long longValueBigDecimalFloor = bigDecimalFloor.longValue(); - boolean isLongExpected = - bigDecimalFloor.equals(bigDecimalDec.valueOf(longValueBigDecimalFloor)); - - boolean decIsLong = dec.isLong(); - long oldDecLong = oldDec.longValue(); - long decLong = dec.longValue(); - if (isLongExpected != decIsLong) { - Assert.fail(); - } - - if (decIsLong) { - if (oldDecLong != decLong) { - Assert.fail(); - } - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomIntValue() { - Random r = new Random(98333); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomIntValue(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomIntValue(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomIntValue(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestIntValue(bigDecimal); - } - } - - @Test - public void testIntValueSpecial() { - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestIntValue(bigDecimal); - } - } - - private void doTestIntValue(BigDecimal bigDecimal) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec = HiveDecimal.create(bigDecimal); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - BigDecimal bigDecimalOldDec = oldDec.bigDecimalValue(); - BigDecimal bigDecimalDec = dec.bigDecimalValue(); - Assert.assertEquals(bigDecimalOldDec, bigDecimalDec); - - BigDecimal bigDecimalFloor = bigDecimalDec.setScale(0, BigDecimal.ROUND_DOWN); - int intValueBigDecimalFloor = bigDecimalFloor.intValue(); - boolean isIntExpected = - bigDecimalFloor.equals(bigDecimalDec.valueOf(intValueBigDecimalFloor)); - - boolean decIsInt = dec.isInt(); - int oldDecInt = oldDec.intValue(); - int decInt = dec.intValue(); - if (isIntExpected != decIsInt) { - Assert.fail(); - } - - if (decIsInt) { - if (oldDecInt != decInt) { - Assert.fail(); - } - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomShortValue() { - Random r = new Random(15); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomShortValue(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomShortValue(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomShortValue(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestShortValue(bigDecimal); - } - } - - @Test - public void testShortValueSpecial() { - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestShortValue(bigDecimal); - } - } - - private void doTestShortValue(BigDecimal bigDecimal) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec = HiveDecimal.create(bigDecimal); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - BigDecimal bigDecimalOldDec = oldDec.bigDecimalValue(); - BigDecimal bigDecimalDec = dec.bigDecimalValue(); - Assert.assertEquals(bigDecimalOldDec, bigDecimalDec); - - BigDecimal bigDecimalFloor = bigDecimalDec.setScale(0, BigDecimal.ROUND_DOWN); - short shortValueBigDecimalFloor = bigDecimalFloor.shortValue(); - boolean isShortExpected = - bigDecimalFloor.equals(bigDecimalDec.valueOf(shortValueBigDecimalFloor)); - - boolean decIsShort = dec.isShort(); - short oldDecShort = oldDec.shortValue(); - short decShort = dec.shortValue(); - if (isShortExpected != decIsShort) { - Assert.fail(); - } - - if (decIsShort) { - if (oldDecShort != decShort) { - Assert.fail(); - } - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomByteValue() { - Random r = new Random(9292); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomByteValue(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomByteValue(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomByteValue(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestByteValue(bigDecimal); - } - } - - @Test - public void testByteValueSpecial() { - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestByteValue(bigDecimal); - } - } - - private void doTestByteValue(BigDecimal bigDecimal) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec = HiveDecimal.create(bigDecimal); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - BigDecimal bigDecimalOldDec = oldDec.bigDecimalValue(); - BigDecimal bigDecimalDec = dec.bigDecimalValue(); - Assert.assertEquals(bigDecimalOldDec, bigDecimalDec); - - BigDecimal bigDecimalFloor = bigDecimalDec.setScale(0, BigDecimal.ROUND_DOWN); - byte byteValueBigDecimalFloor = bigDecimalFloor.byteValue(); - boolean isByteExpected = - bigDecimalFloor.equals(bigDecimalDec.valueOf(byteValueBigDecimalFloor)); - - boolean decIsByte = dec.isByte(); - byte oldDecByte = oldDec.byteValue(); - byte decByte = dec.byteValue(); - if (isByteExpected != decIsByte) { - Assert.fail(); - } - - if (decIsByte) { - if (oldDecByte != decByte) { - Assert.fail(); - } - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomTimestamp() { - Random r = new Random(5476); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomTimestamp(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomTimestamp(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomTimestamp(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestTimestamp(bigDecimal); - } - } - - @Test - public void testTimestampSpecial() { - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestTimestamp(bigDecimal); - } - } - - private void doTestTimestamp(BigDecimal bigDecimal) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec = HiveDecimal.create(bigDecimal); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - Timestamp timestampOldDec = TimestampUtils.decimalToTimestamp(oldDec); - Timestamp timestampDec = TimestampUtils.decimalToTimestamp(dec); - if (timestampOldDec == null) { - Assert.assertTrue(timestampDec == null); - return; - } - if (timestampDec == null) { - return; - } - Assert.assertEquals(timestampOldDec, timestampDec); - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomBigIntegerBytes() { - Random r = new Random(1050); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomBigIntegerBytes(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomBigIntegerBytes(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomBigIntegerBytes(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestBigIntegerBytes(bigDecimal); - } - } - - @Test - public void testBigIntegerBytesSpecial() { - Random r = new Random(1050); - for (BigDecimal bigDecimal : specialBigDecimals) { - int negativeScale = -(0 + r.nextInt(38 + 1)); - bigDecimal = bigDecimal.setScale(negativeScale, BigDecimal.ROUND_HALF_UP); - doTestBigIntegerBytes(bigDecimal); - } - } - - private void doTestBigIntegerBytes(BigDecimal bigDecimal) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - if (oldDec == null) { - return; - } - HiveDecimal dec = HiveDecimal.create(bigDecimal); - assertTrue(dec != null); - dec.validate(); - - //--------------------------------------------------- - BigInteger oldBigInteger = oldDec.unscaledValue(); - int oldScale = oldDec.scale(); - //--------------------------------------------------- - - BigInteger bigInteger = dec.unscaledValue(); - int scale = dec.scale(); - - long[] scratchLongs = new long[HiveDecimal.SCRATCH_LONGS_LEN]; - byte[] scratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; - - int which = 0; - try { - which = 1; - int byteLength = dec.bigIntegerBytes(scratchLongs, scratchBuffer); - byte[] bytes = null; - if (byteLength == 0) { - Assert.fail(); - } else { - bytes = Arrays.copyOf(scratchBuffer, byteLength); - } - - which = 2; - byte[] bytesExpected = bigInteger.toByteArray(); - String bytesExpectedString = displayBytes(bytesExpected, 0, bytesExpected.length); - - if (!StringExpr.equal(bytes, 0, bytes.length, bytesExpected, 0, bytesExpected.length)) { - fail(); - } - - which = 3; - HiveDecimal createFromBigIntegerBytesDec = - HiveDecimal.createFromBigIntegerBytesAndScale( - bytes, 0, bytes.length, scale); - if (!createFromBigIntegerBytesDec.equals(dec)) { - fail(); - } - - } catch (Exception e) { - fail(); - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomToFormatString() { - Random r = new Random(1051); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomToFormatString(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomToFormatString(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomToFormatString(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestToFormatString(r, bigDecimal); - } - } - - @Test - public void testToFormatStringSpecial() { - Random r = new Random(1050); - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestToFormatString(r, bigDecimal); - } - } - - private void doTestToFormatString(Random r, BigDecimal bigDecimal) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec; - if (oldDec == null) { - dec = HiveDecimal.create(bigDecimal); - if (dec != null) { - Assert.fail(); - } - return; - } else { - dec = HiveDecimal.create(bigDecimal); - if (dec == null) { - if (isTenPowerBug(oldDec.toString())) { - return; - } - Assert.fail(); - } - } - dec.validate(); - - // UNDONE: Does this random range need to go as high as 38? - int formatScale = 0 + r.nextInt(38); - - String oldDecFormatString = oldDec.toFormatString(formatScale); - String decFormatString; - if (oldDecFormatString == null) { - decFormatString = dec.toFormatString(formatScale); - if (decFormatString != null) { - Assert.fail(); - } - return; - } else { - decFormatString = dec.toFormatString(formatScale); - if (decFormatString == null) { - if (isTenPowerBug(oldDecFormatString)) { - return; - } - Assert.fail(); - } - } - - if (!oldDecFormatString.equals(decFormatString)) { - fail(); - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomScaleByPowerOfTen() { - Random r = new Random(1052); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomScaleByPowerOfTen(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomScaleByPowerOfTen(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomScaleByPowerOfTen(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestScaleByPowerOfTen(r, bigDecimal); - } - } - - @Test - public void testScaleByPowerOfTenSpecial() { - Random r = new Random(1050); - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestScaleByPowerOfTen(r, bigDecimal); - } - } - - private void doTestScaleByPowerOfTen(Random r, BigDecimal bigDecimal) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec = HiveDecimal.create(bigDecimal); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - HiveDecimalV1 oldPowerDec; - HiveDecimal powerDec; - - for (int power = -(2 * HiveDecimal.MAX_SCALE + 1); - power <= 2 * HiveDecimal.MAX_SCALE + 1; - power++) { - - oldPowerDec = oldDec.scaleByPowerOfTen(power); - boolean isEqual; - if (oldPowerDec == null) { - powerDec = dec.scaleByPowerOfTen(power); - if (powerDec != null) { - Assert.fail(); - } - return; - } else { - String oldPowerDecString = oldPowerDec.toString(); - powerDec = dec.scaleByPowerOfTen(power); - if (powerDec == null) { - if (isTenPowerBug(oldPowerDec.toString())) { - return; - } - Assert.fail(); - continue; - } - powerDec.validate(); - String powerDecString = powerDec.toString(); - isEqual = oldPowerDecString.equals(powerDecString); - if (!isEqual) { - if (oldPowerDecString.equals("0.00000000000000000000000000000000000001") || - oldPowerDecString.equals("-0.00000000000000000000000000000000000001")) { - continue; - } - Assert.fail(); - } - } - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomWriteReadFields() throws Exception { - Random r = new Random(1052); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomWriteReadFields(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomWriteReadFields(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomWriteReadFields(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) throws Exception { - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestWriteReadFields(r, bigDecimal); - } - } - - @Test - public void testWriteReadFieldsSpecial() throws Exception { - Random r = new Random(1050); - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestWriteReadFields(r, bigDecimal); - } - } - - private void doTestWriteReadFields(Random r, BigDecimal bigDecimal) throws IOException { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec = HiveDecimal.create(bigDecimal); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream out = new DataOutputStream(baos); - - HiveDecimalWritable decimalWritableOut = new HiveDecimalWritable(dec); - decimalWritableOut.write(out); - - byte[] valueBytes = baos.toByteArray(); - - ByteArrayInputStream bais = new ByteArrayInputStream(valueBytes); - DataInputStream in = new DataInputStream(bais); - HiveDecimalWritable decimalWritableIn = new HiveDecimalWritable(); - decimalWritableIn.readFields(in); - - Assert.assertEquals(dec, decimalWritableIn.getHiveDecimal()); - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomBigIntegerBytesScaled() throws Exception { - Random r = new Random(1052); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomBigIntegerBytesScaled(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomBigIntegerBytesScaled(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomBigIntegerBytesScaled(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) throws Exception { - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestBigIntegerBytesScaled(r, bigDecimal); - } - } - - @Test - public void testBigIntegerBytesScaledSpecial() throws Exception { - Random r = new Random(1050); - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestBigIntegerBytesScaled(r, bigDecimal); - } - } - - private void doTestBigIntegerBytesScaled(Random r, BigDecimal bigDecimal) throws IOException { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec = HiveDecimal.create(bigDecimal); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - int scale = oldDec.scale(); - int newScale; - if (scale == HiveDecimal.MAX_SCALE) { - newScale = scale; - } else { - newScale = scale + r.nextInt(HiveDecimal.MAX_SCALE - scale); - } - - HiveDecimalV1 oldDecScaled = oldDec.setScale(newScale); - HiveDecimalWritableV1 oldDecScaledWritable = new HiveDecimalWritableV1(oldDecScaled); - - byte[] bytesExpected = oldDecScaledWritable.getInternalStorage(); - - byte[] bytes = dec.bigIntegerBytesScaled(newScale); - - if (!StringExpr.equal(bytes, 0, bytes.length, bytesExpected, 0, bytesExpected.length)) { - Assert.fail(); - } - - HiveDecimalWritableV1 oldDecWritableRetrieve = new HiveDecimalWritableV1(bytesExpected, newScale); - HiveDecimalV1 oldDecRetrieve = oldDecWritableRetrieve.getHiveDecimal(); - Assert.assertTrue(oldDecRetrieve != null); - - HiveDecimal decRetrieve = HiveDecimal.createFromBigIntegerBytesAndScale(bytes, newScale); - Assert.assertTrue(decRetrieve != null); - - Assert.assertEquals(oldDecRetrieve.toString(), decRetrieve.toString()); - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomRoundFloor() { - Random r = new Random(1052); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomRound(r, standardAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_FLOOR); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomRound(r, sparseAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_FLOOR); - } - } - } - - private void doTestRandomRound(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor, int roundingMode) { - - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestRound(r, bigDecimal, roundingMode); - } - } - - @Test - public void testRoundFloorSpecial() { - Random r = new Random(1050); - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestRound(r, bigDecimal, HiveDecimal.ROUND_FLOOR); - } - } - - // Used by all flavors. - private void doTestRound(Random r, BigDecimal bigDecimal, int roundingMode) { - - // Temporarily.... - bigDecimal = bigDecimal.abs(); - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec = HiveDecimal.create(bigDecimal); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - HiveDecimalV1 oldScaledDec; - HiveDecimal scaledDec; - - for (int newScale = -(2 * HiveDecimal.MAX_SCALE + 1); - newScale <= 2 * HiveDecimal.MAX_SCALE + 1; - newScale++) { - - oldScaledDec = oldDec.setScale(newScale, roundingMode); - boolean isEqual; - if (oldScaledDec == null) { - scaledDec = dec.setScale(newScale, roundingMode); - if (scaledDec != null) { - Assert.fail(); - } - return; - } else { - scaledDec = dec.setScale(newScale, roundingMode); - if (scaledDec == null) { - if (isTenPowerBug(oldScaledDec.toString())) { - continue; - } - Assert.fail(); - } - scaledDec.validate(); - isEqual = oldScaledDec.toString().equals(scaledDec.toString()); - if (!isEqual) { - Assert.fail(); - } - } - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomRoundCeiling() { - Random r = new Random(1053); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomRound(r, standardAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_CEILING); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomRound(r, sparseAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_CEILING); - } - } - } - - @Test - public void testRoundCeilingSpecial() { - Random r = new Random(1050); - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestRound(r, bigDecimal, HiveDecimal.ROUND_CEILING); - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomRoundHalfUp() { - Random r = new Random(1053); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomRound(r, standardAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_HALF_UP); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomRound(r, sparseAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_HALF_UP); - } - } - } - - @Test - public void testRoundHalfUpSpecial() { - Random r = new Random(1050); - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestRound(r, bigDecimal, HiveDecimal.ROUND_HALF_UP); - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomRoundHalfEven() { - Random r = new Random(1053); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomRound(r, standardAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_HALF_EVEN); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomRound(r, sparseAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_HALF_EVEN); - } - } - } - - @Test - public void testRoundHalfEvenSpecial() { - Random r = new Random(1050); - for (BigDecimal bigDecimal : specialBigDecimals) { - doTestRound(r, bigDecimal, HiveDecimal.ROUND_HALF_EVEN); - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomCompareTo() { - Random r = new Random(1054); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomCompareTo(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomCompareTo(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomCompareTo(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - - for (BigDecimalPairFlavor bigDecimalPairFlavor : BigDecimalPairFlavor.values()) { - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal[] pair = randHiveBigDecimalPair(r, digitAlphabet, bigDecimalFlavor, bigDecimalPairFlavor); - - doTestCompareTo(r, pair[0], pair[1]); - } - } - } - - @Test - public void testCompareToSpecial() { - Random r = new Random(1050); - for (BigDecimal bigDecimal : specialBigDecimals) { - for (BigDecimal bigDecimal2 : specialBigDecimals) { - doTestCompareTo(r, bigDecimal, bigDecimal2); - } - } - } - - private void doTestCompareTo(Random r, BigDecimal bigDecimal, BigDecimal bigDecimal2) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec = HiveDecimal.create(bigDecimal); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - HiveDecimalV1 oldDec2 = HiveDecimalV1.create(bigDecimal2); - if (oldDec2 != null && isTenPowerBug(oldDec2.toString())) { - return; - } - HiveDecimal dec2 = HiveDecimal.create(bigDecimal2); - if (oldDec2 == null) { - assertTrue(dec2 == null); - return; - } - assertTrue(dec2 != null); - dec.validate(); - - // Verify. - Assert.assertEquals(oldDec.toString(), dec.toString()); - Assert.assertEquals(oldDec2.toString(), dec2.toString()); - - int oldCompareTo; - int compareTo; - - // Same object. - oldCompareTo = oldDec.compareTo(oldDec); - Assert.assertEquals(0, oldCompareTo); - compareTo = dec.compareTo(dec); - Assert.assertEquals(0, compareTo); - - // Two objects. - oldCompareTo = oldDec.compareTo(oldDec2); - compareTo = dec.compareTo(dec2); - Assert.assertEquals(oldCompareTo, compareTo); - - int oldCompareToReverse = oldDec2.compareTo(oldDec); - int compareToReverse = dec2.compareTo(dec); - Assert.assertEquals(oldCompareToReverse, compareToReverse); - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomAdd() { - Random r = new Random(1055); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomAdd(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomAdd(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomAdd(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - - for (BigDecimalPairFlavor bigDecimalPairFlavor : BigDecimalPairFlavor.values()) { - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal[] pair = randHiveBigDecimalPair(r, digitAlphabet, bigDecimalFlavor, bigDecimalPairFlavor); - - doTestAdd(r, pair[0], pair[1]); - } - } - } - - @Test - public void testAddSpecial() { - Random r = new Random(1050); - for (BigDecimal bigDecimal : specialBigDecimals) { - for (BigDecimal bigDecimal2 : specialBigDecimals) { - doTestAdd(r, bigDecimal, bigDecimal2); - } - } - } - - private void doTestAdd(Random r, BigDecimal bigDecimal, BigDecimal bigDecimal2) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec = HiveDecimal.create(bigDecimal); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - HiveDecimalV1 oldDec2 = HiveDecimalV1.create(bigDecimal2); - if (oldDec2 != null && isTenPowerBug(oldDec2.toString())) { - return; - } - HiveDecimal dec2 = HiveDecimal.create(bigDecimal2); - if (oldDec2 == null) { - assertTrue(dec2 == null); - return; - } - assertTrue(dec2 != null); - dec.validate(); - - // Verify. - Assert.assertEquals(oldDec.toString(), dec.toString()); - Assert.assertEquals(oldDec2.toString(), dec2.toString()); - - // Add to self. - HiveDecimalV1 oldAddDec; - HiveDecimal addDec; - - oldAddDec = oldDec.add(oldDec); - boolean isEqual; - if (oldAddDec == null) { - addDec = dec.add(dec); - assertTrue(addDec == null); - return; - } else { - addDec = dec.add(dec); - if (addDec == null) { - if (isTenPowerBug(oldAddDec.toString())) { - return; - } - Assert.fail(); - } - addDec.validate(); - isEqual = oldAddDec.toString().equals(addDec.toString()); - if (!isEqual) { - Assert.fail(); - } - } - - // Add two decimals. - oldAddDec = oldDec.add(oldDec2); - if (oldAddDec == null) { - addDec = dec.add(dec2); - assertTrue(addDec == null); - return; - } else { - addDec = dec.add(dec2); - if (addDec == null) { - if (isTenPowerBug(oldAddDec.toString())) { - return; - } - Assert.fail(); - } - addDec.validate(); - isEqual = oldAddDec.toString().equals(addDec.toString()); - if (!isEqual) { - Assert.fail(); - } - } - - // Add negative self. - - oldAddDec = oldDec.add(oldDec.negate()); - if (oldAddDec == null) { - addDec = dec.add(dec.negate()); - assertTrue(addDec == null); - return; - } else { - addDec = dec.add(dec.negate()); - if (addDec == null) { - if (isTenPowerBug(oldAddDec.toString())) { - return; - } - Assert.fail(); - } - addDec.validate(); - isEqual = oldAddDec.toString().equals(addDec.toString()); - if (!isEqual) { - Assert.fail(); - } - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomSubtract() { - Random r = new Random(1055); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomSubtract(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomSubtract(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomSubtract(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - - for (BigDecimalPairFlavor bigDecimalPairFlavor : BigDecimalPairFlavor.values()) { - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal[] pair = randHiveBigDecimalPair(r, digitAlphabet, bigDecimalFlavor, bigDecimalPairFlavor); - - doTestSubtract(r, pair[0], pair[1]); - } - } - } - - @Test - public void testSubtractSpecial() { - Random r = new Random(1050); - for (BigDecimal bigDecimal : specialBigDecimals) { - for (BigDecimal bigDecimal2 : specialBigDecimals) { - doTestSubtract(r, bigDecimal, bigDecimal2); - } - } - } - - private void doTestSubtract(Random r, BigDecimal bigDecimal, BigDecimal bigDecimal2) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec = HiveDecimal.create(bigDecimal); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - HiveDecimalV1 oldDec2 = HiveDecimalV1.create(bigDecimal2); - if (oldDec2 != null && isTenPowerBug(oldDec2.toString())) { - return; - } - HiveDecimal dec2 = HiveDecimal.create(bigDecimal2); - if (oldDec2 == null) { - assertTrue(dec2 == null); - return; - } - assertTrue(dec2 != null); - dec.validate(); - - // Verify. - Assert.assertEquals(oldDec.toString(), dec.toString()); - Assert.assertEquals(oldDec2.toString(), dec2.toString()); - - // Subtract from self. - HiveDecimalV1 oldSubtractDec; - HiveDecimal subtractDec; - - oldSubtractDec = oldDec.subtract(oldDec); - Assert.assertEquals(0, oldSubtractDec.signum()); - subtractDec = dec.subtract(dec); - Assert.assertEquals(0, subtractDec.signum()); - - boolean isEqual; - oldSubtractDec = oldDec.subtract(oldDec2); - if (oldSubtractDec == null) { - subtractDec = dec.subtract(dec2); - assertTrue(subtractDec == null); - return; - } else { - subtractDec = dec.subtract(dec2); - if (subtractDec == null) { - if (isTenPowerBug(oldSubtractDec.toString())) { - return; - } - Assert.fail(); - } - subtractDec.validate(); - isEqual = oldSubtractDec.toString().equals(subtractDec.toString()); - if (!isEqual) { - Assert.fail(); - } - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomMultiply() { - Random r = new Random(1056); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomMultiply(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomMultiply(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomMultiply(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - - for (BigDecimalPairFlavor bigDecimalPairFlavor : BigDecimalPairFlavor.values()) { - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal[] pair = randHiveBigDecimalPair(r, digitAlphabet, bigDecimalFlavor, bigDecimalPairFlavor); - - doTestMultiply(r, pair[0], pair[1]); - } - } - } - - @Test - public void testMultiplySpecial() { - Random r = new Random(1050); - for (BigDecimal bigDecimal : specialBigDecimals) { - for (BigDecimal bigDecimal2 : specialBigDecimals) { - doTestMultiply(r, bigDecimal, bigDecimal2); - } - } - } - - private void doTestMultiply(Random r, BigDecimal bigDecimal, BigDecimal bigDecimal2) { - - HiveDecimalV1 oldDec = HiveDecimalV1.create(bigDecimal); - if (oldDec != null && isTenPowerBug(oldDec.toString())) { - return; - } - HiveDecimal dec = HiveDecimal.create(bigDecimal); - if (oldDec == null) { - assertTrue(dec == null); - return; - } - assertTrue(dec != null); - dec.validate(); - - HiveDecimalV1 oldDec2 = HiveDecimalV1.create(bigDecimal2); - if (oldDec2 != null && isTenPowerBug(oldDec2.toString())) { - return; - } - HiveDecimal dec2 = HiveDecimal.create(bigDecimal2); - if (oldDec2 == null) { - assertTrue(dec2 == null); - return; - } - assertTrue(dec2 != null); - dec.validate(); - - // Verify. - Assert.assertEquals(oldDec.toString(), dec.toString()); - Assert.assertEquals(oldDec2.toString(), dec2.toString()); - - // Multiply by self. - BigDecimal bigDecimalMultiply = bigDecimal.multiply(bigDecimal); - BigDecimal bigDecimalMultiplyAbs = bigDecimalMultiply.abs(); - String bigDecimalMultiplyAbsString = bigDecimalMultiplyAbs.toString(); - int digits = bigDecimalMultiplyAbsString.indexOf('.') != -1 ? bigDecimalMultiplyAbsString.length() - 1: bigDecimalMultiplyAbsString.length(); - HiveDecimalV1 oldMultiplyDec; - HiveDecimal multiplyDec; - - oldMultiplyDec = oldDec.multiply(oldDec); - boolean isEqual; - if (oldMultiplyDec == null) { - multiplyDec = dec.multiply(dec); - if (multiplyDec != null) { - Assert.fail(); - } - return; - } else { - multiplyDec = dec.multiply(dec); - if (multiplyDec == null) { - if (isTenPowerBug(oldMultiplyDec.toString())) { - return; - } - Assert.fail(); - } - multiplyDec.validate(); - isEqual = oldMultiplyDec.toString().equals(multiplyDec.toString()); - if (!isEqual) { - Assert.fail(); - } - } - - bigDecimalMultiply = bigDecimal.multiply(bigDecimal2); - bigDecimalMultiplyAbs = bigDecimalMultiply.abs(); - bigDecimalMultiplyAbsString = bigDecimalMultiplyAbs.toString(); - digits = bigDecimalMultiplyAbsString.indexOf('.') != -1 ? bigDecimalMultiplyAbsString.length() - 1: bigDecimalMultiplyAbsString.length(); - oldMultiplyDec = oldDec.multiply(oldDec2); - if (oldMultiplyDec == null) { - multiplyDec = dec.multiply(dec2); - if (multiplyDec != null) { - Assert.fail(); - } - return; - } else { - multiplyDec = dec.multiply(dec2); - if (multiplyDec == null) { - if (isTenPowerBug(oldMultiplyDec.toString())) { - return; - } - Assert.fail(); - } - multiplyDec.validate(); - isEqual = oldMultiplyDec.toString().equals(multiplyDec.toString()); - if (!isEqual) { - Assert.fail(); - } - } - } - - //------------------------------------------------------------------------------------------------ - - @Test - public void testRandomDecimal64() { - Random r = new Random(2497); - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - doTestRandomDecimal64(r, standardAlphabet, bigDecimalFlavor); - } - for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { - for (String sparseAlphabet : sparseAlphabets) { - doTestRandomDecimal64(r, sparseAlphabet, bigDecimalFlavor); - } - } - } - - private void doTestRandomDecimal64(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { - - for (int i = 0; i < POUND_FACTOR; i++) { - BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); - - doTestDecimal64(r, bigDecimal); - } - } - - @Test - public void testDecimal64Special() { - Random r = new Random(198); - for (BigDecimal bigDecimal : specialBigDecimals) { - int precision = Math.min(bigDecimal.precision(), HiveDecimalWritable.DECIMAL64_DECIMAL_DIGITS); - int scale = Math.min(bigDecimal.scale(), precision); - doTestDecimal64(r, bigDecimal, precision, scale); - } - } - - private void doTestDecimal64(Random r, BigDecimal inputBigDecimal) { - final int precision = 1 + r.nextInt(HiveDecimalWritable.DECIMAL64_DECIMAL_DIGITS); - assertTrue(HiveDecimalWritable.isPrecisionDecimal64(precision)); - final int scale = r.nextInt(precision + 1); - - doTestDecimal64(r, inputBigDecimal, precision, scale); - } - - private void doTestDecimal64(Random r, BigDecimal inputBigDecimal, int precision, int scale) { - - BigDecimal bigDecimal = inputBigDecimal; - - if (!bigDecimal.equals(BigDecimal.ZERO)) { - while (true) { - bigDecimal = bigDecimal.remainder(BigDecimal.valueOf(10).pow(precision - scale)); - bigDecimal = bigDecimal.setScale(scale, BigDecimal.ROUND_DOWN); - if (!bigDecimal.unscaledValue().equals(BigInteger.ZERO)) { - break; - } - bigDecimal = randHiveBigDecimalNormalRange(r, standardAlphabet); - } - } - - HiveDecimal dec = HiveDecimal.create(bigDecimal); - assertTrue(dec != null); - dec.validate(); - - HiveDecimalWritable decWritable = new HiveDecimalWritable(dec); - - final long decimal64Long = decWritable.serialize64(scale); - assertTrue(decimal64Long <= HiveDecimalWritable.getDecimal64AbsMax(precision)); - HiveDecimalWritable resultWritable = new HiveDecimalWritable(0); - resultWritable.deserialize64(decimal64Long, scale); - - assertEquals(dec, resultWritable.getHiveDecimal()); - } - - //------------------------------------------------------------------------------------------------ - - public static String displayBytes(byte[] bytes, int start, int length) { - StringBuilder sb = new StringBuilder(); - for (int i = start; i < start + length; i++) { - sb.append(String.format("\\%03d", (int) (bytes[i] & 0xff))); - } - return sb.toString(); - } -} diff --git a/storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimalVersion.java b/storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimalVersion.java deleted file mode 100644 index cf80b0c81d3091653f8903bb7c2080c61713f24b..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimalVersion.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -import java.lang.annotation.Annotation; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.lang.reflect.Field; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Arrays; -import java.util.TreeSet; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.math.BigDecimal; -import java.math.BigInteger; - -import org.apache.hadoop.hive.VersionTestBase; -import org.junit.*; - -import static org.junit.Assert.*; - -public class TestHiveDecimalVersion extends VersionTestBase { - - /* - * Validation: - * 1) Substitute class name for "ThisClass". - * 2) Only public fields and methods are versioned. - * 3) Methods compare on [non-]static, return type, name, parameter types, exceptions thrown. - * 4) Fields compare on [non-]static, type, name, value when static - */ - @Test - public void testVerifyHiveDecimalPublicMethodsAndFieldsVersions() throws IllegalAccessException { - - Map versionedClassToNameMap = new HashMap(); - versionedClassToNameMap.put(HiveDecimalV1.class, "HiveDecimal"); - versionedClassToNameMap.put(HiveDecimal.class, "HiveDecimal"); - - doVerifyVersions( - HiveDecimalV1.class, HiveDecimalVersionV1.class, - HiveDecimal.class, HiveDecimalVersionV2.class, - versionedClassToNameMap); - - } -} diff --git a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestBytesColumnVector.java b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestBytesColumnVector.java deleted file mode 100644 index e14abf1006a4aa0e4c83f1e5a914abccbc92d3a1..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestBytesColumnVector.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector; - -import java.util.Arrays; - -import org.junit.Test; -import static org.junit.Assert.*; - -public class TestBytesColumnVector { - @Test - public void testSmallBufferReuse() { - BytesColumnVector col = new BytesColumnVector(); - int smallWriteSize = 1024; - int largeWriteSize = 1024 * 1024 * 2; - - int rowIdx = 0; - int bytesWrittenToBytes1 = 0; - col.reset(); - - // Initial write (small value) - byte[] bytes1 = writeToBytesColumnVector(rowIdx, col, smallWriteSize, (byte) 1); - bytesWrittenToBytes1 += smallWriteSize; - - // Write a large value. This should use a different byte buffer - rowIdx++; - byte[] bytes2 = writeToBytesColumnVector(rowIdx, col, largeWriteSize, (byte) 2); - assertFalse(bytes1 == bytes2); - - // Another small write. smallBuffer should be re-used for this write - rowIdx++; - byte[] bytes3 = writeToBytesColumnVector(rowIdx, col, smallWriteSize, (byte) 1); - bytesWrittenToBytes1 += smallWriteSize; - assertTrue(bytes1 == bytes3); - - // Write another large value. This should use a different byte buffer - rowIdx++; - byte[] bytes4 = writeToBytesColumnVector(rowIdx, col, largeWriteSize, (byte) 3); - assertFalse(bytes1 == bytes4); - assertFalse(bytes2 == bytes4); - - // Eventually enough small writes should result in another buffer getting created - boolean gotNewBuffer = false; - // Test is dependent on getting a new buffer within 1MB. - // This may need to change as the implementation changes. - for (int i = 0; i < 1024; ++i) { - rowIdx++; - byte[] currBytes = writeToBytesColumnVector(rowIdx, col, smallWriteSize, (byte) 1); - if (currBytes == bytes1) { - bytesWrittenToBytes1 += smallWriteSize; - } else { - gotNewBuffer = true; - break; - } - } - - assertTrue(gotNewBuffer); - - // All small writes to the first buffer should be in contiguous memory - for (int i = 0; i < bytesWrittenToBytes1; ++i) { - assertEquals((byte) 1, bytes1[i]); - } - } - - // Write a value to the column vector, and return back the byte buffer used. - private static byte[] writeToBytesColumnVector(int rowIdx, BytesColumnVector col, int writeSize, byte val) { - col.ensureValPreallocated(writeSize); - byte[] bytes = col.getValPreallocatedBytes(); - int startIdx = col.getValPreallocatedStart(); - Arrays.fill(bytes, startIdx, startIdx + writeSize, val); - col.setValPreallocated(rowIdx, writeSize); - return bytes; - } -} diff --git a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestListColumnVector.java b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestListColumnVector.java deleted file mode 100644 index 395d8f5d7520420bd2178f5ab9ee8eab4d2c6487..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestListColumnVector.java +++ /dev/null @@ -1,200 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector; - -import org.junit.Test; - -import java.util.Arrays; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Test for ListColumnVector - */ -public class TestListColumnVector { - - @Test - public void testFlatten() throws Exception { - LongColumnVector col1 = new LongColumnVector(10); - ListColumnVector vector = new ListColumnVector(10, col1); - vector.init(); - - // TEST - repeating NULL & no selection - col1.isRepeating = true; - vector.isRepeating = true; - vector.noNulls = false; - vector.isNull[0] = true; - vector.childCount = 0; - for(int i=0; i < 10; ++i) { - col1.vector[i] = i + 3; - vector.offsets[i] = i; - vector.lengths[i] = 10 + i; - } - vector.flatten(false, null, 10); - // make sure the vector was flattened - assertFalse(vector.isRepeating); - assertFalse(vector.noNulls); - // child isn't flattened, because parent is repeating null - assertTrue(col1.isRepeating); - assertTrue(col1.noNulls); - for(int i=0; i < 10; ++i) { - assertTrue("isNull at " + i, vector.isNull[i]); - } - for(int i=0; i < 10; ++i) { - StringBuilder buf = new StringBuilder(); - vector.stringifyValue(buf, i); - assertEquals("null", buf.toString()); - } - vector.unFlatten(); - assertTrue(col1.isRepeating); - assertTrue(vector.isRepeating); - - // TEST - repeating NULL & selection - Arrays.fill(vector.isNull, 1, 10, false); - int[] sel = new int[]{3, 5, 7}; - vector.flatten(true, sel, 3); - for(int i=1; i < 10; i++) { - assertEquals("failure at " + i, - i == 3 || i == 5 || i == 7, vector.isNull[i]); - } - vector.unFlatten(); - - // TEST - repeating non-NULL & no-selection - vector.noNulls = true; - vector.isRepeating = true; - vector.offsets[0] = 0; - vector.lengths[0] = 3; - vector.childCount = 3; - vector.flatten(false, null, 10); - // make sure the vector was flattened - assertFalse(vector.isRepeating); - assertFalse(vector.noNulls); - assertFalse(col1.isRepeating); - assertFalse(col1.noNulls); - for(int i=0; i < 10; ++i) { - assertEquals("offset at " + i, 0, vector.offsets[i]); - assertEquals("length at " + i, 3, vector.lengths[i]); - } - for(int i=0; i < 10; ++i) { - StringBuilder buf = new StringBuilder(); - vector.stringifyValue(buf, i); - assertEquals("[3, 3, 3]", buf.toString()); - } - vector.unFlatten(); - assertTrue(col1.isRepeating); - assertTrue(col1.noNulls); - assertTrue(vector.isRepeating); - assertTrue(vector.noNulls); - - // TEST - repeating non-NULL & selection - Arrays.fill(vector.offsets, 1, 10, -1); - Arrays.fill(vector.lengths, 1, 10, -1); - Arrays.fill(col1.vector, 1, 10, -1); - vector.flatten(true, sel, 3); - for(int i=1; i < 10; i++) { - if (i == 3 || i == 5 || i == 7) { - assertEquals("failure at " + i, 0, vector.offsets[i]); - assertEquals("failure at " + i, 3, vector.lengths[i]); - } else { - assertEquals("failure at " + i, -1, vector.offsets[i]); - assertEquals("failure at " + i, -1, vector.lengths[i]); - } - } - for(int i=0; i < 3; ++i) { - assertEquals("failure at " + i, 3, col1.vector[i]); - } - for(int i=3; i < 10; ++i) { - assertEquals("failure at " + i, -1, col1.vector[i]); - } - vector.unFlatten(); - - // TEST - reset - vector.reset(); - assertFalse(col1.isRepeating); - assertTrue(col1.noNulls); - assertFalse(vector.isRepeating); - assertTrue(vector.noNulls); - assertEquals(0, vector.childCount); - } - - @Test - public void testSet() throws Exception { - LongColumnVector input1 = new LongColumnVector(10); - ListColumnVector input = new ListColumnVector(10, input1); - input.init(); - LongColumnVector output1 = new LongColumnVector(30); - ListColumnVector output = new ListColumnVector(10, output1); - output.init(); - input.noNulls = false; - input.isNull[6] = true; - input.childCount = 11; - Arrays.fill(output1.vector, -1); - for(int i=0; i < 10; ++i) { - input1.vector[i] = 10 * i; - input.offsets[i] = i; - input.lengths[i] = 2; - output.offsets[i] = i + 2; - output.lengths[i] = 3; - } - output.childCount = 30; - - // copy a null - output.setElement(3, 6, input); - assertEquals(30, output.childCount); - StringBuilder buf = new StringBuilder(); - output.stringifyValue(buf, 3); - assertEquals("null", buf.toString()); - - // copy a value - output.setElement(3, 5, input); - assertEquals(30, output.offsets[3]); - assertEquals(2, output.lengths[3]); - assertEquals(32, output.childCount); - buf = new StringBuilder(); - output.stringifyValue(buf, 3); - assertEquals("[50, 60]", buf.toString()); - - // overwrite a value - output.setElement(3, 4, input); - assertEquals(34, output.childCount); - assertEquals(34, output1.vector.length); - assertEquals(50, output1.vector[30]); - assertEquals(60, output1.vector[31]); - buf = new StringBuilder(); - output.stringifyValue(buf, 3); - assertEquals("[40, 50]", buf.toString()); - - input.reset(); - assertEquals(false, input1.isRepeating); - assertEquals(true, input.noNulls); - output.reset(); - assertEquals(0, output.childCount); - - input.isRepeating = true; - input.offsets[0] = 0; - input.lengths[0] = 10; - output.setElement(2, 7, input); - assertEquals(10, output.childCount); - buf = new StringBuilder(); - output.stringifyValue(buf, 2); - assertEquals("[0, 10, 20, 30, 40, 50, 60, 70, 80, 90]", buf.toString()); - } -} diff --git a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestMapColumnVector.java b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestMapColumnVector.java deleted file mode 100644 index c77c286d54256829f6c95715f5f1bb4ed83c6bb5..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestMapColumnVector.java +++ /dev/null @@ -1,224 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector; - -import org.junit.Test; - -import java.util.Arrays; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Test for MapColumnVector - */ -public class TestMapColumnVector { - - @Test - public void testFlatten() throws Exception { - LongColumnVector col1 = new LongColumnVector(10); - DoubleColumnVector col2 = new DoubleColumnVector(10); - MapColumnVector vector = new MapColumnVector(10, col1, col2); - vector.init(); - - // TEST - repeating NULL & no selection - col1.isRepeating = true; - vector.isRepeating = true; - vector.noNulls = false; - vector.isNull[0] = true; - vector.childCount = 0; - for(int i=0; i < 10; ++i) { - col1.vector[i] = i + 3; - col2.vector[i] = i * 10; - vector.offsets[i] = i; - vector.lengths[i] = 10 + i; - } - vector.flatten(false, null, 10); - // make sure the vector was flattened - assertFalse(vector.isRepeating); - assertFalse(vector.noNulls); - // child isn't flattened, because parent is repeating null - assertTrue(col1.isRepeating); - assertTrue(col1.noNulls); - for(int i=0; i < 10; ++i) { - assertTrue("isNull at " + i, vector.isNull[i]); - } - for(int i=0; i < 10; ++i) { - StringBuilder buf = new StringBuilder(); - vector.stringifyValue(buf, i); - assertEquals("null", buf.toString()); - } - vector.unFlatten(); - assertTrue(col1.isRepeating); - assertTrue(vector.isRepeating); - - // TEST - repeating NULL & selection - Arrays.fill(vector.isNull, 1, 10, false); - int[] sel = new int[]{3, 5, 7}; - vector.flatten(true, sel, 3); - for(int i=1; i < 10; i++) { - assertEquals("failure at " + i, - i == 3 || i == 5 || i == 7, vector.isNull[i]); - } - vector.unFlatten(); - - // TEST - repeating non-NULL & no-selection - vector.noNulls = true; - vector.isRepeating = true; - vector.offsets[0] = 0; - vector.lengths[0] = 3; - vector.childCount = 3; - vector.flatten(false, null, 10); - // make sure the vector was flattened - assertFalse(vector.isRepeating); - assertFalse(vector.noNulls); - assertFalse(col1.isRepeating); - assertFalse(col1.noNulls); - assertFalse(col2.isRepeating); - assertFalse(col2.noNulls); - for(int i=0; i < 10; ++i) { - assertEquals("offset at " + i, 0, vector.offsets[i]); - assertEquals("length at " + i, 3, vector.lengths[i]); - } - for(int i=0; i < 10; ++i) { - StringBuilder buf = new StringBuilder(); - vector.stringifyValue(buf, i); - assertEquals("[{\"key\": 3, \"value\": 0.0}," + - " {\"key\": 3, \"value\": 10.0}," + - " {\"key\": 3, \"value\": 20.0}]", buf.toString()); - } - vector.unFlatten(); - assertTrue(col1.isRepeating); - assertTrue(col1.noNulls); - assertTrue(vector.isRepeating); - assertFalse(col2.isRepeating); - assertTrue(col2.noNulls); - assertTrue(vector.noNulls); - - // TEST - repeating non-NULL & selection - Arrays.fill(vector.offsets, 1, 10, -1); - Arrays.fill(vector.lengths, 1, 10, -1); - Arrays.fill(col1.vector, 1, 10, -1); - vector.flatten(true, sel, 3); - for(int i=1; i < 10; i++) { - if (i == 3 || i == 5 || i == 7) { - assertEquals("failure at " + i, 0, vector.offsets[i]); - assertEquals("failure at " + i, 3, vector.lengths[i]); - } else { - assertEquals("failure at " + i, -1, vector.offsets[i]); - assertEquals("failure at " + i, -1, vector.lengths[i]); - } - } - for(int i=0; i < 3; ++i) { - assertEquals("failure at " + i, 3, col1.vector[i]); - } - for(int i=3; i < 10; ++i) { - assertEquals("failure at " + i, -1, col1.vector[i]); - } - vector.unFlatten(); - - // TEST - reset - vector.reset(); - assertFalse(col1.isRepeating); - assertTrue(col1.noNulls); - assertFalse(col2.isRepeating); - assertTrue(col2.noNulls); - assertFalse(vector.isRepeating); - assertTrue(vector.noNulls); - assertEquals(0, vector.childCount); - } - - @Test - public void testSet() throws Exception { - LongColumnVector input1 = new LongColumnVector(10); - DoubleColumnVector input2 = new DoubleColumnVector(10); - MapColumnVector input = new MapColumnVector(10, input1, input2); - input.init(); - LongColumnVector output1 = new LongColumnVector(30); - DoubleColumnVector output2 = new DoubleColumnVector(30); - MapColumnVector output = new MapColumnVector(10, output1, output2); - output.init(); - input.noNulls = false; - input.isNull[6] = true; - input.childCount = 11; - Arrays.fill(output1.vector, -1); - for(int i=0; i < 10; ++i) { - input1.vector[i] = 10 * i; - input2.vector[i] = 100 * i; - input.offsets[i] = i; - input.lengths[i] = 2; - output.offsets[i] = i + 2; - output.lengths[i] = 3; - } - output.childCount = 30; - - // copy a null - output.setElement(3, 6, input); - assertEquals(30, output.childCount); - StringBuilder buf = new StringBuilder(); - output.stringifyValue(buf, 3); - assertEquals("null", buf.toString()); - - // copy a value - output.setElement(3, 5, input); - assertEquals(30, output.offsets[3]); - assertEquals(2, output.lengths[3]); - assertEquals(32, output.childCount); - buf = new StringBuilder(); - output.stringifyValue(buf, 3); - assertEquals("[{\"key\": 50, \"value\": 500.0}," + - " {\"key\": 60, \"value\": 600.0}]", buf.toString()); - - // overwrite a value - output.setElement(3, 4, input); - assertEquals(34, output.childCount); - assertEquals(34, output1.vector.length); - assertEquals(50, output1.vector[30]); - assertEquals(60, output1.vector[31]); - buf = new StringBuilder(); - output.stringifyValue(buf, 3); - assertEquals("[{\"key\": 40, \"value\": 400.0}," + - " {\"key\": 50, \"value\": 500.0}]", buf.toString()); - - input.reset(); - assertEquals(false, input1.isRepeating); - assertEquals(true, input.noNulls); - output.reset(); - assertEquals(0, output.childCount); - - input.isRepeating = true; - input.offsets[0] = 0; - input.lengths[0] = 10; - output.setElement(2, 7, input); - assertEquals(10, output.childCount); - buf = new StringBuilder(); - output.stringifyValue(buf, 2); - assertEquals("[{\"key\": 0, \"value\": 0.0}," + - " {\"key\": 10, \"value\": 100.0}," + - " {\"key\": 20, \"value\": 200.0}," + - " {\"key\": 30, \"value\": 300.0}," + - " {\"key\": 40, \"value\": 400.0}," + - " {\"key\": 50, \"value\": 500.0}," + - " {\"key\": 60, \"value\": 600.0}," + - " {\"key\": 70, \"value\": 700.0}," + - " {\"key\": 80, \"value\": 800.0}," + - " {\"key\": 90, \"value\": 900.0}]", buf.toString()); - } -} diff --git a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java deleted file mode 100644 index c175ed08c73e4c8d177b0015ddabae413f6f4c21..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector; - -import org.junit.Test; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.sql.Timestamp; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Test for StructColumnVector - */ -public class TestStructColumnVector { - - @Test - public void testFlatten() throws Exception { - LongColumnVector col1 = new LongColumnVector(10); - LongColumnVector col2 = new LongColumnVector(10); - StructColumnVector vector = new StructColumnVector(10, col1, col2); - vector.init(); - col1.isRepeating = true; - for(int i=0; i < 10; ++i) { - col1.vector[i] = i; - col2.vector[i] = 2 * i; - } - vector.flatten(false, null, 10); - assertFalse(col1.isRepeating); - for(int i=0; i < 10; ++i) { - assertEquals("col1 at " + i, 0, col1.vector[i]); - assertEquals("col2 at " + i, 2 * i, col2.vector[i]); - } - vector.unFlatten(); - assertTrue(col1.isRepeating); - for(int i=0; i < 10; ++i) { - StringBuilder buf = new StringBuilder(); - vector.stringifyValue(buf, i); - assertEquals("[0, " + (2 * i) + "]", buf.toString()); - } - vector.reset(); - assertFalse(col1.isRepeating); - } - - @Test - public void testSet() throws Exception { - LongColumnVector input1 = new LongColumnVector(10); - LongColumnVector input2 = new LongColumnVector(10); - StructColumnVector input = new StructColumnVector(10, input1, input2); - input.init(); - LongColumnVector output1 = new LongColumnVector(10); - LongColumnVector output2 = new LongColumnVector(10); - StructColumnVector output = new StructColumnVector(10, output1, output2); - output.init(); - input1.isRepeating = true; - input2.noNulls = false; - input2.isNull[5] = true; - input.noNulls = false; - input.isNull[6] = true; - for(int i=0; i < 10; ++i) { - input1.vector[i] = i + 1; - input2.vector[i] = i + 2; - } - output.setElement(3, 6, input); - StringBuilder buf = new StringBuilder(); - output.stringifyValue(buf, 3); - assertEquals("null", buf.toString()); - output.setElement(3, 5, input); - buf = new StringBuilder(); - output.stringifyValue(buf, 3); - assertEquals("[1, null]", buf.toString()); - output.setElement(3, 4, input); - buf = new StringBuilder(); - output.stringifyValue(buf, 3); - assertEquals("[1, 6]", buf.toString()); - input.reset(); - assertEquals(false, input1.isRepeating); - assertEquals(true, input.noNulls); - } - - @Test - public void testStringify() throws IOException { - VectorizedRowBatch batch = new VectorizedRowBatch(2); - LongColumnVector x1 = new LongColumnVector(); - TimestampColumnVector x2 = new TimestampColumnVector(); - StructColumnVector x = new StructColumnVector(1024, x1, x2); - BytesColumnVector y = new BytesColumnVector(); - batch.cols[0] = x; - batch.cols[1] = y; - batch.reset(); - Timestamp ts = Timestamp.valueOf("2000-01-01 00:00:00"); - for(int r=0; r < 10; ++r) { - batch.size += 1; - x1.vector[r] = 3 * r; - ts.setTime(ts.getTime() + 1000); - x2.set(r, ts); - byte[] buffer = ("value " + r).getBytes(StandardCharsets.UTF_8); - y.setRef(r, buffer, 0, buffer.length); - } - final String EXPECTED = ("Column vector types: 0:STRUCT, 1:BYTES\n" + - "[[0, 2000-01-01 00:00:01.0], \"value 0\"]\n" + - "[[3, 2000-01-01 00:00:02.0], \"value 1\"]\n" + - "[[6, 2000-01-01 00:00:03.0], \"value 2\"]\n" + - "[[9, 2000-01-01 00:00:04.0], \"value 3\"]\n" + - "[[12, 2000-01-01 00:00:05.0], \"value 4\"]\n" + - "[[15, 2000-01-01 00:00:06.0], \"value 5\"]\n" + - "[[18, 2000-01-01 00:00:07.0], \"value 6\"]\n" + - "[[21, 2000-01-01 00:00:08.0], \"value 7\"]\n" + - "[[24, 2000-01-01 00:00:09.0], \"value 8\"]\n" + - "[[27, 2000-01-01 00:00:10.0], \"value 9\"]"); - assertEquals(EXPECTED, batch.toString()); - } -} diff --git a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java deleted file mode 100644 index 6e5d5c8cfb8a8d33eba568e8180c5620652b0ec2..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector; - -import org.junit.Test; - -import java.io.PrintWriter; -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.sql.Timestamp; -import java.util.Date; -import java.util.Random; - -import org.apache.hadoop.hive.common.type.RandomTypeUtil; - -import static org.junit.Assert.*; - -/** - * Test for ListColumnVector - */ -public class TestTimestampColumnVector { - - private static int TEST_COUNT = 5000; - - private static int fake = 0; - - @Test - public void testSaveAndRetrieve() throws Exception { - - Random r = new Random(1234); - TimestampColumnVector timestampColVector = new TimestampColumnVector(); - Timestamp[] randTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE]; - - for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { - Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r); - randTimestamps[i] = randTimestamp; - timestampColVector.set(i, randTimestamp); - } - for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { - Timestamp retrievedTimestamp = timestampColVector.asScratchTimestamp(i); - Timestamp randTimestamp = randTimestamps[i]; - if (!retrievedTimestamp.equals(randTimestamp)) { - assertTrue(false); - } - } - } - - @Test - public void testTimestampCompare() throws Exception { - Random r = new Random(1234); - TimestampColumnVector timestampColVector = new TimestampColumnVector(); - Timestamp[] randTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE]; - Timestamp[] candTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE]; - int[] compareToLeftRights = new int[VectorizedRowBatch.DEFAULT_SIZE]; - int[] compareToRightLefts = new int[VectorizedRowBatch.DEFAULT_SIZE]; - - for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { - Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r); - randTimestamps[i] = randTimestamp; - timestampColVector.set(i, randTimestamp); - Timestamp candTimestamp = RandomTypeUtil.getRandTimestamp(r); - candTimestamps[i] = candTimestamp; - compareToLeftRights[i] = candTimestamp.compareTo(randTimestamp); - compareToRightLefts[i] = randTimestamp.compareTo(candTimestamp); - } - - for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { - Timestamp retrievedTimestamp = timestampColVector.asScratchTimestamp(i); - Timestamp randTimestamp = randTimestamps[i]; - if (!retrievedTimestamp.equals(randTimestamp)) { - assertTrue(false); - } - Timestamp candTimestamp = candTimestamps[i]; - int compareToLeftRight = timestampColVector.compareTo(candTimestamp, i); - if (compareToLeftRight != compareToLeftRights[i]) { - assertTrue(false); - } - int compareToRightLeft = timestampColVector.compareTo(i, candTimestamp); - if (compareToRightLeft != compareToRightLefts[i]) { - assertTrue(false); - } - } - } - - /* - @Test - public void testGenerate() throws Exception { - PrintWriter writer = new PrintWriter("/Users/you/timestamps.txt"); - Random r = new Random(18485); - for (int i = 0; i < 25; i++) { - Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r); - writer.println(randTimestamp.toString()); - } - for (int i = 0; i < 25; i++) { - Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r, 1965, 2025); - writer.println(randTimestamp.toString()); - } - writer.close(); - } - */ -} diff --git a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestUnionColumnVector.java b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestUnionColumnVector.java deleted file mode 100644 index c378cd41a46bbb3d242547db944dacf90dde9e56..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestUnionColumnVector.java +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Test for StructColumnVector - */ -public class TestUnionColumnVector { - - @Test - public void testFlatten() throws Exception { - LongColumnVector col1 = new LongColumnVector(10); - LongColumnVector col2 = new LongColumnVector(10); - UnionColumnVector vector = new UnionColumnVector(10, col1, col2); - vector.init(); - col1.isRepeating = true; - for(int i=0; i < 10; ++i) { - vector.tags[i] = i % 2; - col1.vector[i] = i; - col2.vector[i] = 2 * i; - } - vector.flatten(false, null, 10); - assertFalse(col1.isRepeating); - for(int i=0; i < 10; ++i) { - assertEquals(i % 2, vector.tags[i]); - assertEquals("col1 at " + i, 0, col1.vector[i]); - assertEquals("col2 at " + i, 2 * i, col2.vector[i]); - } - vector.unFlatten(); - assertTrue(col1.isRepeating); - for(int i=0; i < 10; ++i) { - StringBuilder buf = new StringBuilder(); - vector.stringifyValue(buf, i); - assertEquals("{\"tag\": " + (i % 2) + ", \"value\": " + - (i % 2 == 0 ? 0 : 2 * i) + "}", buf.toString()); - } - vector.reset(); - assertFalse(col1.isRepeating); - } - - @Test - public void testSet() throws Exception { - LongColumnVector input1 = new LongColumnVector(10); - LongColumnVector input2 = new LongColumnVector(10); - UnionColumnVector input = new UnionColumnVector(10, input1, input2); - input.init(); - LongColumnVector output1 = new LongColumnVector(10); - LongColumnVector output2 = new LongColumnVector(10); - UnionColumnVector output = new UnionColumnVector(10, output1, output2); - output.init(); - input1.isRepeating = true; - for(int i=0; i < 10; ++i) { - input.tags[i] = i % 2; - input1.vector[i] = i + 1; - input2.vector[i] = i + 2; - } - output.setElement(3, 4, input); - StringBuilder buf = new StringBuilder(); - output.stringifyValue(buf, 3); - assertEquals("{\"tag\": 0, \"value\": 1}", buf.toString()); - input.noNulls = false; - input.isNull[5] = true; - output.setElement(3, 5, input); - buf = new StringBuilder(); - output.stringifyValue(buf, 3); - assertEquals("null", buf.toString()); - input.reset(); - assertEquals(false, input1.isRepeating); - assertEquals(true, input.noNulls); - } -} diff --git a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestStringExpr.java b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestStringExpr.java deleted file mode 100644 index 63c210ad4a8f75f65da2fe449891bc4ee33b4970..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestStringExpr.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions; - -import org.junit.Test; - -import java.nio.charset.StandardCharsets; - -import static org.junit.Assert.*; - -public class TestStringExpr { - @Test - public void test() throws Exception { - StringExpr.Finder pattern = compile("pattern"); - assertNotNull(pattern); - - StringExpr.Finder patternOneChar = compile("g"); - assertNotNull(patternOneChar); - - StringExpr.Finder patternZero = compile(""); - assertNotNull(patternZero); - - String input1 = "string that contains a patterN..."; - String input2 = "string that contains a pattern..."; - String input3 = "pattern at the start of a string"; - String input4 = "string that ends with a pattern"; - - assertEquals("Testing invalid match", -1, find(pattern, input1)); - assertEquals("Testing valid match", 23, find(pattern, input2)); - assertEquals("Testing single-character match", 5, find(patternOneChar, input1)); - assertEquals("Testing zero-length pattern", 0, find(patternZero, input1)); - assertEquals("Testing match at start of string", 0, find(pattern, input3)); - assertEquals("Testing match at end of string", 24, find(pattern, input4)); - } - - private StringExpr.Finder compile(String pattern) { - return StringExpr.compile(pattern.getBytes(StandardCharsets.UTF_8)); - } - - private int find(StringExpr.Finder finder, String string) { - byte[] bytes = string.getBytes(StandardCharsets.UTF_8); - return finder.find(bytes, 0, bytes.length); - } -} \ No newline at end of file diff --git a/storage-api/src/test/org/apache/hadoop/hive/ql/util/TestJavaDataModel.java b/storage-api/src/test/org/apache/hadoop/hive/ql/util/TestJavaDataModel.java deleted file mode 100644 index 9295b899acb7cad38aa52720f83612688177ce14..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/ql/util/TestJavaDataModel.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.util; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertSame; - -public final class TestJavaDataModel { - - private static final String DATA_MODEL_PROPERTY = "sun.arch.data.model"; - - private String previousModelSetting; - - @Before - public void setUp() throws Exception { - previousModelSetting = System.getProperty(DATA_MODEL_PROPERTY); - } - - @After - public void tearDown() throws Exception { - if (previousModelSetting != null) { - System.setProperty(DATA_MODEL_PROPERTY, previousModelSetting); - } else { - System.clearProperty(DATA_MODEL_PROPERTY); - } - } - - @Test - public void testGetDoesNotReturnNull() throws Exception { - JavaDataModel model = JavaDataModel.get(); - assertNotNull(model); - } - - @Test - public void testGetModelForSystemWhenSetTo32() throws Exception { - System.setProperty(DATA_MODEL_PROPERTY, "32"); - assertSame(JavaDataModel.JAVA32, JavaDataModel.getModelForSystem()); - } - - @Test - public void testGetModelForSystemWhenSetTo64() throws Exception { - System.setProperty(DATA_MODEL_PROPERTY, "64"); - assertSame(JavaDataModel.JAVA64, JavaDataModel.getModelForSystem()); - } - - @Test - public void testGetModelForSystemWhenSetToUnknown() throws Exception { - System.setProperty(DATA_MODEL_PROPERTY, "unknown"); - assertSame(JavaDataModel.JAVA64, JavaDataModel.getModelForSystem()); - } - - @Test - public void testGetModelForSystemWhenUndefined() throws Exception { - System.clearProperty(DATA_MODEL_PROPERTY); - assertSame(JavaDataModel.JAVA64, JavaDataModel.getModelForSystem()); - } -} \ No newline at end of file diff --git a/storage-api/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java b/storage-api/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java deleted file mode 100644 index b8f5472edd78e1c66a24368285ff20ccb47b891a..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.io; - -import org.junit.*; -import static org.junit.Assert.*; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.ArrayList; - -import org.apache.hadoop.hive.common.type.HiveDecimal; - -/** - * Unit tests for tsting the fast allocation-free conversion - * between HiveDecimalWritable and Decimal128 - */ -public class TestHiveDecimalWritable { - - @Test - public void testHiveDecimalWritable() { - - HiveDecimalWritable decWritable; - - HiveDecimal nullDec = null; - decWritable = new HiveDecimalWritable(nullDec); - assertTrue(!decWritable.isSet()); - decWritable = new HiveDecimalWritable("1"); - assertTrue(decWritable.isSet()); - - // UNDONE: more! - } -} - diff --git a/storage-api/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritableVersion.java b/storage-api/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritableVersion.java deleted file mode 100644 index 8e0f396fe0ae240c6df2dda1e31a768976132e83..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritableVersion.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.io; - -import java.lang.annotation.Annotation; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.lang.reflect.Field; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Arrays; -import java.util.TreeSet; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.math.BigDecimal; -import java.math.BigInteger; - -import org.apache.hadoop.hive.VersionTestBase; -import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.common.type.HiveDecimalV1; -import org.junit.*; - -import static org.junit.Assert.*; - -public class TestHiveDecimalWritableVersion extends VersionTestBase { - - /* - * Validation: - * 1) Substitute class name for "ThisClass". - * 2) Only public fields and methods are versioned. - * 3) Methods compare on [non-]static, return type, name, parameter types, exceptions thrown. - * 4) Fields compare on [non-]static, type, name, value when static - */ - @Test - public void testVerifyHiveDecimalWritablePublicMethodsAndFieldsVersions() throws IllegalAccessException { - - Map versionedClassToNameMap = new HashMap(); - versionedClassToNameMap.put(HiveDecimalV1.class, "HiveDecimal"); - versionedClassToNameMap.put(HiveDecimal.class, "HiveDecimal"); - versionedClassToNameMap.put(HiveDecimalWritableV1.class, "HiveDecimalWritable"); - versionedClassToNameMap.put(HiveDecimalWritable.class, "HiveDecimalWritable"); - - doVerifyVersions( - HiveDecimalWritableV1.class, HiveDecimalWritableVersionV1.class, - HiveDecimalWritable.class, HiveDecimalWritableVersionV2.class, - versionedClassToNameMap); - - } -} diff --git a/storage-api/src/test/org/apache/hive/common/util/TestBloomFilter.java b/storage-api/src/test/org/apache/hive/common/util/TestBloomFilter.java deleted file mode 100644 index cd1fa08f9a929c7885cc508fe11d6635d62a236d..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hive/common/util/TestBloomFilter.java +++ /dev/null @@ -1,715 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.common.util; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.util.ArrayList; -import java.util.Random; - -import org.junit.Assert; -import org.junit.Test; - -/** - * - */ -public class TestBloomFilter { - private static final int COUNT = 100; - Random rand = new Random(123); - - @Test(expected = IllegalArgumentException.class) - public void testBloomIllegalArg1() { - BloomFilter bf = new BloomFilter(0, 0); - } - - @Test(expected = IllegalArgumentException.class) - public void testBloomIllegalArg2() { - BloomFilter bf = new BloomFilter(0, 0.1); - } - - @Test(expected = IllegalArgumentException.class) - public void testBloomIllegalArg3() { - BloomFilter bf = new BloomFilter(1, 0.0); - } - - @Test(expected = IllegalArgumentException.class) - public void testBloomIllegalArg4() { - BloomFilter bf = new BloomFilter(1, 1.0); - } - - @Test(expected = IllegalArgumentException.class) - public void testBloomIllegalArg5() { - BloomFilter bf = new BloomFilter(-1, -1); - } - - - @Test - public void testBloomNumBits() { - assertEquals(0, BloomFilter.optimalNumOfBits(0, 0)); - assertEquals(0, BloomFilter.optimalNumOfBits(0, 1)); - assertEquals(0, BloomFilter.optimalNumOfBits(1, 1)); - assertEquals(7, BloomFilter.optimalNumOfBits(1, 0.03)); - assertEquals(72, BloomFilter.optimalNumOfBits(10, 0.03)); - assertEquals(729, BloomFilter.optimalNumOfBits(100, 0.03)); - assertEquals(7298, BloomFilter.optimalNumOfBits(1000, 0.03)); - assertEquals(72984, BloomFilter.optimalNumOfBits(10000, 0.03)); - assertEquals(729844, BloomFilter.optimalNumOfBits(100000, 0.03)); - assertEquals(7298440, BloomFilter.optimalNumOfBits(1000000, 0.03)); - assertEquals(6235224, BloomFilter.optimalNumOfBits(1000000, 0.05)); - assertEquals(1870567268, BloomFilter.optimalNumOfBits(300000000, 0.05)); - assertEquals(1437758756, BloomFilter.optimalNumOfBits(300000000, 0.1)); - assertEquals(432808512, BloomFilter.optimalNumOfBits(300000000, 0.5)); - assertEquals(1393332198, BloomFilter.optimalNumOfBits(3000000000L, 0.8)); - assertEquals(657882327, BloomFilter.optimalNumOfBits(3000000000L, 0.9)); - assertEquals(0, BloomFilter.optimalNumOfBits(3000000000L, 1)); - } - - @Test - public void testBloomNumHashFunctions() { - assertEquals(1, BloomFilter.optimalNumOfHashFunctions(-1, -1)); - assertEquals(1, BloomFilter.optimalNumOfHashFunctions(0, 0)); - assertEquals(1, BloomFilter.optimalNumOfHashFunctions(10, 0)); - assertEquals(1, BloomFilter.optimalNumOfHashFunctions(10, 10)); - assertEquals(7, BloomFilter.optimalNumOfHashFunctions(10, 100)); - assertEquals(1, BloomFilter.optimalNumOfHashFunctions(100, 100)); - assertEquals(1, BloomFilter.optimalNumOfHashFunctions(1000, 100)); - assertEquals(1, BloomFilter.optimalNumOfHashFunctions(10000, 100)); - assertEquals(1, BloomFilter.optimalNumOfHashFunctions(100000, 100)); - assertEquals(1, BloomFilter.optimalNumOfHashFunctions(1000000, 100)); - } - - @Test - public void testBloomFilterBytes() { - BloomFilter bf = new BloomFilter(10000); - byte[] val = new byte[]{1, 2, 3}; - byte[] val1 = new byte[]{1, 2, 3, 4}; - byte[] val2 = new byte[]{1, 2, 3, 4, 5}; - byte[] val3 = new byte[]{1, 2, 3, 4, 5, 6}; - - assertEquals(false, bf.test(val)); - assertEquals(false, bf.test(val1)); - assertEquals(false, bf.test(val2)); - assertEquals(false, bf.test(val3)); - bf.add(val); - assertEquals(true, bf.test(val)); - assertEquals(false, bf.test(val1)); - assertEquals(false, bf.test(val2)); - assertEquals(false, bf.test(val3)); - bf.add(val1); - assertEquals(true, bf.test(val)); - assertEquals(true, bf.test(val1)); - assertEquals(false, bf.test(val2)); - assertEquals(false, bf.test(val3)); - bf.add(val2); - assertEquals(true, bf.test(val)); - assertEquals(true, bf.test(val1)); - assertEquals(true, bf.test(val2)); - assertEquals(false, bf.test(val3)); - bf.add(val3); - assertEquals(true, bf.test(val)); - assertEquals(true, bf.test(val1)); - assertEquals(true, bf.test(val2)); - assertEquals(true, bf.test(val3)); - - byte[] randVal = new byte[COUNT]; - for (int i = 0; i < COUNT; i++) { - rand.nextBytes(randVal); - bf.add(randVal); - } - // last value should be present - assertEquals(true, bf.test(randVal)); - // most likely this value should not exist - randVal[0] = 0; - randVal[1] = 0; - randVal[2] = 0; - randVal[3] = 0; - randVal[4] = 0; - assertEquals(false, bf.test(randVal)); - - assertEquals(7800, bf.sizeInBytes()); - } - - @Test - public void testBloomFilterByte() { - BloomFilter bf = new BloomFilter(10000); - byte val = Byte.MIN_VALUE; - byte val1 = 1; - byte val2 = 2; - byte val3 = Byte.MAX_VALUE; - - assertEquals(false, bf.testLong(val)); - assertEquals(false, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val); - assertEquals(true, bf.testLong(val)); - assertEquals(false, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val1); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val2); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(true, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val3); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(true, bf.testLong(val2)); - assertEquals(true, bf.testLong(val3)); - - byte randVal = 0; - for (int i = 0; i < COUNT; i++) { - randVal = (byte) rand.nextInt(Byte.MAX_VALUE); - bf.addLong(randVal); - } - // last value should be present - assertEquals(true, bf.testLong(randVal)); - // most likely this value should not exist - assertEquals(false, bf.testLong((byte) -120)); - - assertEquals(7800, bf.sizeInBytes()); - } - - @Test - public void testBloomFilterInt() { - BloomFilter bf = new BloomFilter(10000); - int val = Integer.MIN_VALUE; - int val1 = 1; - int val2 = 2; - int val3 = Integer.MAX_VALUE; - - assertEquals(false, bf.testLong(val)); - assertEquals(false, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val); - assertEquals(true, bf.testLong(val)); - assertEquals(false, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val1); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val2); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(true, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val3); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(true, bf.testLong(val2)); - assertEquals(true, bf.testLong(val3)); - - int randVal = 0; - for (int i = 0; i < COUNT; i++) { - randVal = rand.nextInt(); - bf.addLong(randVal); - } - // last value should be present - assertEquals(true, bf.testLong(randVal)); - // most likely this value should not exist - assertEquals(false, bf.testLong(-120)); - - assertEquals(7800, bf.sizeInBytes()); - } - - @Test - public void testBloomFilterLong() { - BloomFilter bf = new BloomFilter(10000); - long val = Long.MIN_VALUE; - long val1 = 1; - long val2 = 2; - long val3 = Long.MAX_VALUE; - - assertEquals(false, bf.testLong(val)); - assertEquals(false, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val); - assertEquals(true, bf.testLong(val)); - assertEquals(false, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val1); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val2); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(true, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val3); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(true, bf.testLong(val2)); - assertEquals(true, bf.testLong(val3)); - - long randVal = 0; - for (int i = 0; i < COUNT; i++) { - randVal = rand.nextLong(); - bf.addLong(randVal); - } - // last value should be present - assertEquals(true, bf.testLong(randVal)); - // most likely this value should not exist - assertEquals(false, bf.testLong(-120)); - - assertEquals(7800, bf.sizeInBytes()); - } - - @Test - public void testBloomFilterFloat() { - BloomFilter bf = new BloomFilter(10000); - float val = Float.MIN_VALUE; - float val1 = 1.1f; - float val2 = 2.2f; - float val3 = Float.MAX_VALUE; - - assertEquals(false, bf.testDouble(val)); - assertEquals(false, bf.testDouble(val1)); - assertEquals(false, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val); - assertEquals(true, bf.testDouble(val)); - assertEquals(false, bf.testDouble(val1)); - assertEquals(false, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val1); - assertEquals(true, bf.testDouble(val)); - assertEquals(true, bf.testDouble(val1)); - assertEquals(false, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val2); - assertEquals(true, bf.testDouble(val)); - assertEquals(true, bf.testDouble(val1)); - assertEquals(true, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val3); - assertEquals(true, bf.testDouble(val)); - assertEquals(true, bf.testDouble(val1)); - assertEquals(true, bf.testDouble(val2)); - assertEquals(true, bf.testDouble(val3)); - - float randVal = 0; - for (int i = 0; i < COUNT; i++) { - randVal = rand.nextFloat(); - bf.addDouble(randVal); - } - // last value should be present - assertEquals(true, bf.testDouble(randVal)); - // most likely this value should not exist - assertEquals(false, bf.testDouble(-120.2f)); - - assertEquals(7800, bf.sizeInBytes()); - } - - @Test - public void testBloomFilterDouble() { - BloomFilter bf = new BloomFilter(10000); - double val = Double.MIN_VALUE; - double val1 = 1.1d; - double val2 = 2.2d; - double val3 = Double.MAX_VALUE; - - assertEquals(false, bf.testDouble(val)); - assertEquals(false, bf.testDouble(val1)); - assertEquals(false, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val); - assertEquals(true, bf.testDouble(val)); - assertEquals(false, bf.testDouble(val1)); - assertEquals(false, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val1); - assertEquals(true, bf.testDouble(val)); - assertEquals(true, bf.testDouble(val1)); - assertEquals(false, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val2); - assertEquals(true, bf.testDouble(val)); - assertEquals(true, bf.testDouble(val1)); - assertEquals(true, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val3); - assertEquals(true, bf.testDouble(val)); - assertEquals(true, bf.testDouble(val1)); - assertEquals(true, bf.testDouble(val2)); - assertEquals(true, bf.testDouble(val3)); - - double randVal = 0; - for (int i = 0; i < COUNT; i++) { - randVal = rand.nextDouble(); - bf.addDouble(randVal); - } - // last value should be present - assertEquals(true, bf.testDouble(randVal)); - // most likely this value should not exist - assertEquals(false, bf.testDouble(-120.2d)); - - assertEquals(7800, bf.sizeInBytes()); - } - - @Test - public void testBloomFilterString() { - BloomFilter bf = new BloomFilter(100000); - String val = "bloo"; - String val1 = "bloom fil"; - String val2 = "bloom filter"; - String val3 = "cuckoo filter"; - - assertEquals(false, bf.testString(val)); - assertEquals(false, bf.testString(val1)); - assertEquals(false, bf.testString(val2)); - assertEquals(false, bf.testString(val3)); - bf.addString(val); - assertEquals(true, bf.testString(val)); - assertEquals(false, bf.testString(val1)); - assertEquals(false, bf.testString(val2)); - assertEquals(false, bf.testString(val3)); - bf.addString(val1); - assertEquals(true, bf.testString(val)); - assertEquals(true, bf.testString(val1)); - assertEquals(false, bf.testString(val2)); - assertEquals(false, bf.testString(val3)); - bf.addString(val2); - assertEquals(true, bf.testString(val)); - assertEquals(true, bf.testString(val1)); - assertEquals(true, bf.testString(val2)); - assertEquals(false, bf.testString(val3)); - bf.addString(val3); - assertEquals(true, bf.testString(val)); - assertEquals(true, bf.testString(val1)); - assertEquals(true, bf.testString(val2)); - assertEquals(true, bf.testString(val3)); - - long randVal = 0; - for (int i = 0; i < COUNT; i++) { - randVal = rand.nextLong(); - bf.addString(Long.toString(randVal)); - } - // last value should be present - assertEquals(true, bf.testString(Long.toString(randVal))); - // most likely this value should not exist - assertEquals(false, bf.testString(Long.toString(-120))); - - assertEquals(77944, bf.sizeInBytes()); - } - - @Test - public void testMerge() { - BloomFilter bf = new BloomFilter(10000); - String val = "bloo"; - String val1 = "bloom fil"; - String val2 = "bloom filter"; - String val3 = "cuckoo filter"; - bf.addString(val); - bf.addString(val1); - bf.addString(val2); - bf.addString(val3); - - BloomFilter bf2 = new BloomFilter(10000); - String v = "2_bloo"; - String v1 = "2_bloom fil"; - String v2 = "2_bloom filter"; - String v3 = "2_cuckoo filter"; - bf2.addString(v); - bf2.addString(v1); - bf2.addString(v2); - bf2.addString(v3); - - assertEquals(true, bf.testString(val)); - assertEquals(true, bf.testString(val1)); - assertEquals(true, bf.testString(val2)); - assertEquals(true, bf.testString(val3)); - assertEquals(false, bf.testString(v)); - assertEquals(false, bf.testString(v1)); - assertEquals(false, bf.testString(v2)); - assertEquals(false, bf.testString(v3)); - - bf.merge(bf2); - - assertEquals(true, bf.testString(val)); - assertEquals(true, bf.testString(val1)); - assertEquals(true, bf.testString(val2)); - assertEquals(true, bf.testString(val3)); - assertEquals(true, bf.testString(v)); - assertEquals(true, bf.testString(v1)); - assertEquals(true, bf.testString(v2)); - assertEquals(true, bf.testString(v3)); - } - - @Test - public void testSerialize() throws Exception { - BloomFilter bf1 = new BloomFilter(10000); - String[] inputs = { - "bloo", - "bloom fil", - "bloom filter", - "cuckoo filter", - }; - - for (String val : inputs) { - bf1.addString(val); - } - - // Serialize/deserialize - ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); - BloomFilter.serialize(bytesOut, bf1); - ByteArrayInputStream bytesIn = new ByteArrayInputStream(bytesOut.toByteArray()); - BloomFilter bf2 = BloomFilter.deserialize(bytesIn); - - for (String val : inputs) { - assertEquals("Testing bf1 with " + val, true, bf1.testString(val)); - assertEquals("Testing bf2 with " + val, true, bf2.testString(val)); - } - } - - @Test - public void testMergeBloomFilterBytes() throws Exception { - BloomFilter bf1 = new BloomFilter(10000); - BloomFilter bf2 = new BloomFilter(10000); - - String[] inputs1 = { - "bloo", - "bloom fil", - "bloom filter", - "cuckoo filter", - }; - - String[] inputs2 = { - "2_bloo", - "2_bloom fil", - "2_bloom filter", - "2_cuckoo filter", - }; - - for (String val : inputs1) { - bf1.addString(val); - } - for (String val : inputs2) { - bf2.addString(val); - } - - ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); - BloomFilter.serialize(bytesOut, bf1); - byte[] bf1Bytes = bytesOut.toByteArray(); - bytesOut.reset(); - BloomFilter.serialize(bytesOut, bf1); - byte[] bf2Bytes = bytesOut.toByteArray(); - - // Merge bytes - BloomFilter.mergeBloomFilterBytes( - bf1Bytes, 0, bf1Bytes.length, - bf2Bytes, 0, bf2Bytes.length); - - // Deserialize and test - ByteArrayInputStream bytesIn = new ByteArrayInputStream(bf1Bytes, 0, bf1Bytes.length); - BloomFilter bfMerged = BloomFilter.deserialize(bytesIn); - // All values should pass test - for (String val : inputs1) { - bfMerged.addString(val); - } - for (String val : inputs2) { - bfMerged.addString(val); - } - } - - @Test - public void testMergeBloomFilterBytesFailureCases() throws Exception { - BloomFilter bf1 = new BloomFilter(1000); - BloomFilter bf2 = new BloomFilter(200); - // Create bloom filter with same number of bits, but different # hash functions - long[] bits = new long[bf1.getBitSet().length]; - BloomFilter bf3 = new BloomFilter(bits, bf1.getNumHashFunctions() + 1); - - // Serialize to bytes - ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); - BloomFilter.serialize(bytesOut, bf1); - byte[] bf1Bytes = bytesOut.toByteArray(); - - bytesOut.reset(); - BloomFilter.serialize(bytesOut, bf2); - byte[] bf2Bytes = bytesOut.toByteArray(); - - bytesOut.reset(); - BloomFilter.serialize(bytesOut, bf3); - byte[] bf3Bytes = bytesOut.toByteArray(); - - try { - // this should fail - BloomFilter.mergeBloomFilterBytes( - bf1Bytes, 0, bf1Bytes.length, - bf2Bytes, 0, bf2Bytes.length); - Assert.fail("Expected exception not encountered"); - } catch (IllegalArgumentException err) { - // expected - } - - try { - // this should fail - BloomFilter.mergeBloomFilterBytes( - bf1Bytes, 0, bf1Bytes.length, - bf3Bytes, 0, bf3Bytes.length); - Assert.fail("Expected exception not encountered"); - } catch (IllegalArgumentException err) { - // expected - } - } - - @Test - public void testFpp1K() { - int size = 1000; - BloomFilter bf = new BloomFilter(size); - int fp = 0; - for (int i = 0; i < size; i++) { - bf.addLong(i); - } - - for (int i = 0; i < size; i++) { - assertTrue(bf.testLong(i)); - } - - for (int i = 0; i < size; i++) { - int probe = rand.nextInt(); - // out of range probes - if ((probe > size) || (probe < 0)) { - if (bf.testLong(probe)) { - fp++; - } - } - } - - double actualFpp = (double) fp / (double) size; - double expectedFpp = bf.DEFAULT_FPP; - if (actualFpp < expectedFpp) { - assertTrue(actualFpp != 0.0); - } else { - assertEquals(expectedFpp, actualFpp, 0.005); - } - } - - @Test - public void testFpp10K() { - int size = 10_000; - BloomFilter bf = new BloomFilter(size); - int fp = 0; - for (int i = 0; i < size; i++) { - bf.addLong(i); - } - - for (int i = 0; i < size; i++) { - assertTrue(bf.testLong(i)); - } - - for (int i = 0; i < size; i++) { - int probe = rand.nextInt(); - // out of range probes - if ((probe > size) || (probe < 0)) { - if (bf.testLong(probe)) { - fp++; - } - } - } - - double actualFpp = (double) fp / (double) size; - double expectedFpp = bf.DEFAULT_FPP; - if (actualFpp < expectedFpp) { - assertTrue(actualFpp != 0.0); - } else { - assertEquals(expectedFpp, actualFpp, 0.005); - } - } - - @Test - public void testFpp1M() { - int size = 1_000_000; - BloomFilter bf = new BloomFilter(size); - int fp = 0; - for (int i = 0; i < size; i++) { - bf.addLong(i); - } - - for (int i = 0; i < size; i++) { - assertTrue(bf.testLong(i)); - } - - for (int i = 0; i < size; i++) { - int probe = rand.nextInt(); - // out of range probes - if ((probe > size) || (probe < 0)) { - if (bf.testLong(probe)) { - fp++; - } - } - } - - double actualFpp = (double) fp / (double) size; - double expectedFpp = bf.DEFAULT_FPP; - if (actualFpp < expectedFpp) { - assertTrue(actualFpp != 0.0); - } else { - assertEquals(expectedFpp, actualFpp, 0.005); - } - } - - @Test - public void testFpp10M() { - int size = 10_000_000; - BloomFilter bf = new BloomFilter(size); - int fp = 0; - for (int i = 0; i < size; i++) { - bf.addLong(i); - } - - for (int i = 0; i < size; i++) { - assertTrue(bf.testLong(i)); - } - - for (int i = 0; i < size; i++) { - int probe = rand.nextInt(); - // out of range probes - if ((probe > size) || (probe < 0)) { - if (bf.testLong(probe)) { - fp++; - } - } - } - - double actualFpp = (double) fp / (double) size; - double expectedFpp = bf.DEFAULT_FPP; - if (actualFpp < expectedFpp) { - assertTrue(actualFpp != 0.0); - } else { - assertEquals(expectedFpp, actualFpp, 0.005); - } - } -} diff --git a/storage-api/src/test/org/apache/hive/common/util/TestBloomKFilter.java b/storage-api/src/test/org/apache/hive/common/util/TestBloomKFilter.java deleted file mode 100644 index 159fab2dd0b354ba36427aedc1bb79f5af12ce90..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hive/common/util/TestBloomKFilter.java +++ /dev/null @@ -1,699 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.common.util; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.util.Random; - -import org.junit.Assert; -import org.junit.Test; - -/** - * - */ -public class TestBloomKFilter { - private static final int COUNT = 100; - Random rand = new Random(123); - // bloom-1 is known to have higher fpp, to make tests pass give room for another 3% - private final double deltaError = 0.03; - - @Test(expected = IllegalArgumentException.class) - public void testBloomIllegalArg1() { - BloomKFilter bf = new BloomKFilter(0); - } - - @Test - public void testBloomNumBits() { - assertEquals(0, BloomKFilter.optimalNumOfBits(0, 0)); - assertEquals(0, BloomKFilter.optimalNumOfBits(0, 1)); - assertEquals(0, BloomKFilter.optimalNumOfBits(1, 1)); - assertEquals(7, BloomKFilter.optimalNumOfBits(1, 0.03)); - assertEquals(72, BloomKFilter.optimalNumOfBits(10, 0.03)); - assertEquals(729, BloomKFilter.optimalNumOfBits(100, 0.03)); - assertEquals(7298, BloomKFilter.optimalNumOfBits(1000, 0.03)); - assertEquals(72984, BloomKFilter.optimalNumOfBits(10000, 0.03)); - assertEquals(729844, BloomKFilter.optimalNumOfBits(100000, 0.03)); - assertEquals(7298440, BloomKFilter.optimalNumOfBits(1000000, 0.03)); - assertEquals(6235224, BloomKFilter.optimalNumOfBits(1000000, 0.05)); - assertEquals(1870567268, BloomKFilter.optimalNumOfBits(300000000, 0.05)); - assertEquals(1437758756, BloomKFilter.optimalNumOfBits(300000000, 0.1)); - assertEquals(432808512, BloomKFilter.optimalNumOfBits(300000000, 0.5)); - assertEquals(1393332198, BloomKFilter.optimalNumOfBits(3000000000L, 0.8)); - assertEquals(657882327, BloomKFilter.optimalNumOfBits(3000000000L, 0.9)); - assertEquals(0, BloomKFilter.optimalNumOfBits(3000000000L, 1)); - - BloomKFilter bloomKFilter = new BloomKFilter(40); - assertEquals(8, bloomKFilter.getBitSet().length); - assertEquals(bloomKFilter.getNumBits(), bloomKFilter.getBitSize()); - } - - @Test - public void testBloomNumHashFunctions() { - assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(-1, -1)); - assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(0, 0)); - assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(10, 0)); - assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(10, 10)); - assertEquals(7, BloomKFilter.optimalNumOfHashFunctions(10, 100)); - assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(100, 100)); - assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(1000, 100)); - assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(10000, 100)); - assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(100000, 100)); - assertEquals(1, BloomKFilter.optimalNumOfHashFunctions(1000000, 100)); - } - - @Test - public void testBloomKFilterBytes() { - BloomKFilter bf = new BloomKFilter(10000); - byte[] val = new byte[]{1, 2, 3}; - byte[] val1 = new byte[]{1, 2, 3, 4}; - byte[] val2 = new byte[]{1, 2, 3, 4, 5}; - byte[] val3 = new byte[]{1, 2, 3, 4, 5, 6}; - - assertEquals(false, bf.test(val)); - assertEquals(false, bf.test(val1)); - assertEquals(false, bf.test(val2)); - assertEquals(false, bf.test(val3)); - bf.add(val); - assertEquals(true, bf.test(val)); - assertEquals(false, bf.test(val1)); - assertEquals(false, bf.test(val2)); - assertEquals(false, bf.test(val3)); - bf.add(val1); - assertEquals(true, bf.test(val)); - assertEquals(true, bf.test(val1)); - assertEquals(false, bf.test(val2)); - assertEquals(false, bf.test(val3)); - bf.add(val2); - assertEquals(true, bf.test(val)); - assertEquals(true, bf.test(val1)); - assertEquals(true, bf.test(val2)); - assertEquals(false, bf.test(val3)); - bf.add(val3); - assertEquals(true, bf.test(val)); - assertEquals(true, bf.test(val1)); - assertEquals(true, bf.test(val2)); - assertEquals(true, bf.test(val3)); - - byte[] randVal = new byte[COUNT]; - for (int i = 0; i < COUNT; i++) { - rand.nextBytes(randVal); - bf.add(randVal); - } - // last value should be present - assertEquals(true, bf.test(randVal)); - // most likely this value should not exist - randVal[0] = 0; - randVal[1] = 0; - randVal[2] = 0; - randVal[3] = 0; - randVal[4] = 0; - assertEquals(false, bf.test(randVal)); - - assertEquals(7808, bf.sizeInBytes()); - } - - @Test - public void testBloomKFilterByte() { - BloomKFilter bf = new BloomKFilter(10000); - byte val = Byte.MIN_VALUE; - byte val1 = 1; - byte val2 = 2; - byte val3 = Byte.MAX_VALUE; - - assertEquals(false, bf.testLong(val)); - assertEquals(false, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val); - assertEquals(true, bf.testLong(val)); - assertEquals(false, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val1); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val2); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(true, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val3); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(true, bf.testLong(val2)); - assertEquals(true, bf.testLong(val3)); - - byte randVal = 0; - for (int i = 0; i < COUNT; i++) { - randVal = (byte) rand.nextInt(Byte.MAX_VALUE); - bf.addLong(randVal); - } - // last value should be present - assertEquals(true, bf.testLong(randVal)); - // most likely this value should not exist - assertEquals(false, bf.testLong((byte) -120)); - - assertEquals(7808, bf.sizeInBytes()); - } - - @Test - public void testBloomKFilterInt() { - BloomKFilter bf = new BloomKFilter(10000); - int val = Integer.MIN_VALUE; - int val1 = 1; - int val2 = 2; - int val3 = Integer.MAX_VALUE; - - assertEquals(false, bf.testLong(val)); - assertEquals(false, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val); - assertEquals(true, bf.testLong(val)); - assertEquals(false, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val1); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val2); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(true, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val3); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(true, bf.testLong(val2)); - assertEquals(true, bf.testLong(val3)); - - int randVal = 0; - for (int i = 0; i < COUNT; i++) { - randVal = rand.nextInt(); - bf.addLong(randVal); - } - // last value should be present - assertEquals(true, bf.testLong(randVal)); - // most likely this value should not exist - assertEquals(false, bf.testLong(-120)); - - assertEquals(7808, bf.sizeInBytes()); - } - - @Test - public void testBloomKFilterLong() { - BloomKFilter bf = new BloomKFilter(10000); - long val = Long.MIN_VALUE; - long val1 = 1; - long val2 = 2; - long val3 = Long.MAX_VALUE; - - assertEquals(false, bf.testLong(val)); - assertEquals(false, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val); - assertEquals(true, bf.testLong(val)); - assertEquals(false, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val1); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(false, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val2); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(true, bf.testLong(val2)); - assertEquals(false, bf.testLong(val3)); - bf.addLong(val3); - assertEquals(true, bf.testLong(val)); - assertEquals(true, bf.testLong(val1)); - assertEquals(true, bf.testLong(val2)); - assertEquals(true, bf.testLong(val3)); - - long randVal = 0; - for (int i = 0; i < COUNT; i++) { - randVal = rand.nextLong(); - bf.addLong(randVal); - } - // last value should be present - assertEquals(true, bf.testLong(randVal)); - // most likely this value should not exist - assertEquals(false, bf.testLong(-120)); - - assertEquals(7808, bf.sizeInBytes()); - } - - @Test - public void testBloomKFilterFloat() { - BloomKFilter bf = new BloomKFilter(10000); - float val = Float.MIN_VALUE; - float val1 = 1.1f; - float val2 = 2.2f; - float val3 = Float.MAX_VALUE; - - assertEquals(false, bf.testDouble(val)); - assertEquals(false, bf.testDouble(val1)); - assertEquals(false, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val); - assertEquals(true, bf.testDouble(val)); - assertEquals(false, bf.testDouble(val1)); - assertEquals(false, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val1); - assertEquals(true, bf.testDouble(val)); - assertEquals(true, bf.testDouble(val1)); - assertEquals(false, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val2); - assertEquals(true, bf.testDouble(val)); - assertEquals(true, bf.testDouble(val1)); - assertEquals(true, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val3); - assertEquals(true, bf.testDouble(val)); - assertEquals(true, bf.testDouble(val1)); - assertEquals(true, bf.testDouble(val2)); - assertEquals(true, bf.testDouble(val3)); - - float randVal = 0; - for (int i = 0; i < COUNT; i++) { - randVal = rand.nextFloat(); - bf.addDouble(randVal); - } - // last value should be present - assertEquals(true, bf.testDouble(randVal)); - // most likely this value should not exist - assertEquals(false, bf.testDouble(-120.2f)); - - assertEquals(7808, bf.sizeInBytes()); - } - - @Test - public void testBloomKFilterDouble() { - BloomKFilter bf = new BloomKFilter(10000); - double val = Double.MIN_VALUE; - double val1 = 1.1d; - double val2 = 2.2d; - double val3 = Double.MAX_VALUE; - - assertEquals(false, bf.testDouble(val)); - assertEquals(false, bf.testDouble(val1)); - assertEquals(false, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val); - assertEquals(true, bf.testDouble(val)); - assertEquals(false, bf.testDouble(val1)); - assertEquals(false, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val1); - assertEquals(true, bf.testDouble(val)); - assertEquals(true, bf.testDouble(val1)); - assertEquals(false, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val2); - assertEquals(true, bf.testDouble(val)); - assertEquals(true, bf.testDouble(val1)); - assertEquals(true, bf.testDouble(val2)); - assertEquals(false, bf.testDouble(val3)); - bf.addDouble(val3); - assertEquals(true, bf.testDouble(val)); - assertEquals(true, bf.testDouble(val1)); - assertEquals(true, bf.testDouble(val2)); - assertEquals(true, bf.testDouble(val3)); - - double randVal = 0; - for (int i = 0; i < COUNT; i++) { - randVal = rand.nextDouble(); - bf.addDouble(randVal); - } - // last value should be present - assertEquals(true, bf.testDouble(randVal)); - // most likely this value should not exist - assertEquals(false, bf.testDouble(-120.2d)); - - assertEquals(7808, bf.sizeInBytes()); - } - - @Test - public void testBloomKFilterString() { - BloomKFilter bf = new BloomKFilter(100000); - String val = "bloo"; - String val1 = "bloom fil"; - String val2 = "bloom filter"; - String val3 = "cuckoo filter"; - - assertEquals(false, bf.testString(val)); - assertEquals(false, bf.testString(val1)); - assertEquals(false, bf.testString(val2)); - assertEquals(false, bf.testString(val3)); - bf.addString(val); - assertEquals(true, bf.testString(val)); - assertEquals(false, bf.testString(val1)); - assertEquals(false, bf.testString(val2)); - assertEquals(false, bf.testString(val3)); - bf.addString(val1); - assertEquals(true, bf.testString(val)); - assertEquals(true, bf.testString(val1)); - assertEquals(false, bf.testString(val2)); - assertEquals(false, bf.testString(val3)); - bf.addString(val2); - assertEquals(true, bf.testString(val)); - assertEquals(true, bf.testString(val1)); - assertEquals(true, bf.testString(val2)); - assertEquals(false, bf.testString(val3)); - bf.addString(val3); - assertEquals(true, bf.testString(val)); - assertEquals(true, bf.testString(val1)); - assertEquals(true, bf.testString(val2)); - assertEquals(true, bf.testString(val3)); - - long randVal = 0; - for (int i = 0; i < COUNT; i++) { - randVal = rand.nextLong(); - bf.addString(Long.toString(randVal)); - } - // last value should be present - assertEquals(true, bf.testString(Long.toString(randVal))); - // most likely this value should not exist - assertEquals(false, bf.testString(Long.toString(-120))); - - assertEquals(77952, bf.sizeInBytes()); - } - - @Test - public void testMerge() { - BloomKFilter bf = new BloomKFilter(10000); - String val = "bloo"; - String val1 = "bloom fil"; - String val2 = "bloom filter"; - String val3 = "cuckoo filter"; - bf.addString(val); - bf.addString(val1); - bf.addString(val2); - bf.addString(val3); - - BloomKFilter bf2 = new BloomKFilter(10000); - String v = "2_bloo"; - String v1 = "2_bloom fil"; - String v2 = "2_bloom filter"; - String v3 = "2_cuckoo filter"; - bf2.addString(v); - bf2.addString(v1); - bf2.addString(v2); - bf2.addString(v3); - - assertEquals(true, bf.testString(val)); - assertEquals(true, bf.testString(val1)); - assertEquals(true, bf.testString(val2)); - assertEquals(true, bf.testString(val3)); - assertEquals(false, bf.testString(v)); - assertEquals(false, bf.testString(v1)); - assertEquals(false, bf.testString(v2)); - assertEquals(false, bf.testString(v3)); - - bf.merge(bf2); - - assertEquals(true, bf.testString(val)); - assertEquals(true, bf.testString(val1)); - assertEquals(true, bf.testString(val2)); - assertEquals(true, bf.testString(val3)); - assertEquals(true, bf.testString(v)); - assertEquals(true, bf.testString(v1)); - assertEquals(true, bf.testString(v2)); - assertEquals(true, bf.testString(v3)); - } - - @Test - public void testSerialize() throws Exception { - BloomKFilter bf1 = new BloomKFilter(10000); - String[] inputs = { - "bloo", - "bloom fil", - "bloom filter", - "cuckoo filter", - }; - - for (String val : inputs) { - bf1.addString(val); - } - - // Serialize/deserialize - ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); - BloomKFilter.serialize(bytesOut, bf1); - ByteArrayInputStream bytesIn = new ByteArrayInputStream(bytesOut.toByteArray()); - BloomKFilter bf2 = BloomKFilter.deserialize(bytesIn); - - for (String val : inputs) { - assertEquals("Testing bf1 with " + val, true, bf1.testString(val)); - assertEquals("Testing bf2 with " + val, true, bf2.testString(val)); - } - } - - @Test - public void testMergeBloomKFilterBytes() throws Exception { - BloomKFilter bf1 = new BloomKFilter(10000); - BloomKFilter bf2 = new BloomKFilter(10000); - - String[] inputs1 = { - "bloo", - "bloom fil", - "bloom filter", - "cuckoo filter", - }; - - String[] inputs2 = { - "2_bloo", - "2_bloom fil", - "2_bloom filter", - "2_cuckoo filter", - }; - - for (String val : inputs1) { - bf1.addString(val); - } - for (String val : inputs2) { - bf2.addString(val); - } - - ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); - BloomKFilter.serialize(bytesOut, bf1); - byte[] bf1Bytes = bytesOut.toByteArray(); - bytesOut.reset(); - BloomKFilter.serialize(bytesOut, bf1); - byte[] bf2Bytes = bytesOut.toByteArray(); - - // Merge bytes - BloomKFilter.mergeBloomFilterBytes( - bf1Bytes, 0, bf1Bytes.length, - bf2Bytes, 0, bf2Bytes.length); - - // Deserialize and test - ByteArrayInputStream bytesIn = new ByteArrayInputStream(bf1Bytes, 0, bf1Bytes.length); - BloomKFilter bfMerged = BloomKFilter.deserialize(bytesIn); - // All values should pass test - for (String val : inputs1) { - bfMerged.addString(val); - } - for (String val : inputs2) { - bfMerged.addString(val); - } - } - - @Test - public void testMergeBloomKFilterBytesFailureCases() throws Exception { - BloomKFilter bf1 = new BloomKFilter(1000); - BloomKFilter bf2 = new BloomKFilter(200); - // Create bloom filter with same number of bits, but different # hash functions - long[] bits = new long[bf1.getBitSet().length]; - BloomKFilter bf3 = new BloomKFilter(bits, bf1.getNumHashFunctions() + 1); - - // Serialize to bytes - ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); - BloomKFilter.serialize(bytesOut, bf1); - byte[] bf1Bytes = bytesOut.toByteArray(); - - bytesOut.reset(); - BloomKFilter.serialize(bytesOut, bf2); - byte[] bf2Bytes = bytesOut.toByteArray(); - - bytesOut.reset(); - BloomKFilter.serialize(bytesOut, bf3); - byte[] bf3Bytes = bytesOut.toByteArray(); - - try { - // this should fail - BloomKFilter.mergeBloomFilterBytes( - bf1Bytes, 0, bf1Bytes.length, - bf2Bytes, 0, bf2Bytes.length); - Assert.fail("Expected exception not encountered"); - } catch (IllegalArgumentException err) { - // expected - } - - try { - // this should fail - BloomKFilter.mergeBloomFilterBytes( - bf1Bytes, 0, bf1Bytes.length, - bf3Bytes, 0, bf3Bytes.length); - Assert.fail("Expected exception not encountered"); - } catch (IllegalArgumentException err) { - // expected - } - } - - @Test - public void testFpp1K() { - int size = 1000; - BloomKFilter bf = new BloomKFilter(size); - int fp = 0; - for (int i = 0; i < size; i++) { - bf.addLong(i); - } - - for (int i = 0; i < size; i++) { - assertTrue(bf.testLong(i)); - } - - for (int i = 0; i < size; i++) { - int probe = rand.nextInt(); - // out of range probes - if ((probe > size) || (probe < 0)) { - if (bf.testLong(probe)) { - fp++; - } - } - } - - double actualFpp = (double) fp / (double) size; - double expectedFpp = bf.DEFAULT_FPP; - if (actualFpp < expectedFpp) { - assertTrue(actualFpp != 0.0); - } else { - assertEquals(expectedFpp, actualFpp, deltaError); - } - } - - @Test - public void testFpp10K() { - int size = 10_000; - BloomKFilter bf = new BloomKFilter(size); - int fp = 0; - for (int i = 0; i < size; i++) { - bf.addLong(i); - } - - for (int i = 0; i < size; i++) { - assertTrue(bf.testLong(i)); - } - - for (int i = 0; i < size; i++) { - int probe = rand.nextInt(); - // out of range probes - if ((probe > size) || (probe < 0)) { - if (bf.testLong(probe)) { - fp++; - } - } - } - - double actualFpp = (double) fp / (double) size; - double expectedFpp = bf.DEFAULT_FPP; - if (actualFpp < expectedFpp) { - assertTrue(actualFpp != 0.0); - } else { - assertEquals(expectedFpp, actualFpp, deltaError); - } - } - - @Test - public void testFpp1M() { - int size = 1_000_000; - BloomKFilter bf = new BloomKFilter(size); - int fp = 0; - for (int i = 0; i < size; i++) { - bf.addLong(i); - } - - for (int i = 0; i < size; i++) { - assertTrue(bf.testLong(i)); - } - - for (int i = 0; i < size; i++) { - int probe = rand.nextInt(); - // out of range probes - if ((probe > size) || (probe < 0)) { - if (bf.testLong(probe)) { - fp++; - } - } - } - - double actualFpp = (double) fp / (double) size; - double expectedFpp = bf.DEFAULT_FPP; - if (actualFpp < expectedFpp) { - assertTrue(actualFpp != 0.0); - } else { - assertEquals(expectedFpp, actualFpp, deltaError); - } - } - - @Test - public void testFpp10M() { - int size = 10_000_000; - BloomKFilter bf = new BloomKFilter(size); - int fp = 0; - for (int i = 0; i < size; i++) { - bf.addLong(i); - } - - for (int i = 0; i < size; i++) { - assertTrue(bf.testLong(i)); - } - - for (int i = 0; i < size; i++) { - int probe = rand.nextInt(); - // out of range probes - if ((probe > size) || (probe < 0)) { - if (bf.testLong(probe)) { - fp++; - } - } - } - - double actualFpp = (double) fp / (double) size; - double expectedFpp = bf.DEFAULT_FPP; - if (actualFpp < expectedFpp) { - assertTrue(actualFpp != 0.0); - } else { - assertEquals(expectedFpp, actualFpp, deltaError); - } - } -} diff --git a/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java b/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java deleted file mode 100644 index 391ee425cccbe0f7969c17c8ab76d68f4e6f4de2..0000000000000000000000000000000000000000 --- a/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java +++ /dev/null @@ -1,248 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.common.util; - -import static org.junit.Assert.assertEquals; -import org.apache.hive.common.util.Murmur3.IncrementalHash32; - -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hashing; - -import org.junit.Test; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.Arrays; -import java.util.Random; - -/** - * Tests for Murmur3 variants. - */ -public class TestMurmur3 { - - @Test - public void testHashCodesM3_32_string() { - String key = "test"; - int seed = 123; - HashFunction hf = Hashing.murmur3_32(seed); - int hc1 = hf.hashBytes(key.getBytes()).asInt(); - int hc2 = Murmur3.hash32(key.getBytes(), key.getBytes().length, seed); - assertEquals(hc1, hc2); - - key = "testkey"; - hc1 = hf.hashBytes(key.getBytes()).asInt(); - hc2 = Murmur3.hash32(key.getBytes(), key.getBytes().length, seed); - assertEquals(hc1, hc2); - } - - @Test - public void testHashCodesM3_32_ints() { - int seed = 123; - Random rand = new Random(seed); - HashFunction hf = Hashing.murmur3_32(seed); - for (int i = 0; i < 1000; i++) { - int val = rand.nextInt(); - byte[] data = ByteBuffer.allocate(4).putInt(val).array(); - int hc1 = hf.hashBytes(data).asInt(); - int hc2 = Murmur3.hash32(data, data.length, seed); - assertEquals(hc1, hc2); - } - } - - @Test - public void testHashCodesM3_32_longs() { - int seed = 123; - Random rand = new Random(seed); - HashFunction hf = Hashing.murmur3_32(seed); - for (int i = 0; i < 1000; i++) { - long val = rand.nextLong(); - byte[] data = ByteBuffer.allocate(8).putLong(val).array(); - int hc1 = hf.hashBytes(data).asInt(); - int hc2 = Murmur3.hash32(data, data.length, seed); - assertEquals(hc1, hc2); - } - } - - @Test - public void testHashCodesM3_32_double() { - int seed = 123; - Random rand = new Random(seed); - HashFunction hf = Hashing.murmur3_32(seed); - for (int i = 0; i < 1000; i++) { - double val = rand.nextDouble(); - byte[] data = ByteBuffer.allocate(8).putDouble(val).array(); - int hc1 = hf.hashBytes(data).asInt(); - int hc2 = Murmur3.hash32(data, data.length, seed); - assertEquals(hc1, hc2); - } - } - - @Test - public void testHashCodesM3_128_string() { - String key = "test"; - int seed = 123; - HashFunction hf = Hashing.murmur3_128(seed); - // guava stores the hashcodes in little endian order - ByteBuffer buf = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN); - buf.put(hf.hashBytes(key.getBytes()).asBytes()); - buf.flip(); - long gl1 = buf.getLong(); - long gl2 = buf.getLong(8); - long[] hc = Murmur3.hash128(key.getBytes(), 0, key.getBytes().length, seed); - long m1 = hc[0]; - long m2 = hc[1]; - assertEquals(gl1, m1); - assertEquals(gl2, m2); - - key = "testkey128_testkey128"; - buf = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN); - buf.put(hf.hashBytes(key.getBytes()).asBytes()); - buf.flip(); - gl1 = buf.getLong(); - gl2 = buf.getLong(8); - byte[] keyBytes = key.getBytes(); - hc = Murmur3.hash128(keyBytes, 0, keyBytes.length, seed); - m1 = hc[0]; - m2 = hc[1]; - assertEquals(gl1, m1); - assertEquals(gl2, m2); - - byte[] offsetKeyBytes = new byte[keyBytes.length + 35]; - Arrays.fill(offsetKeyBytes, (byte) -1); - System.arraycopy(keyBytes, 0, offsetKeyBytes, 35, keyBytes.length); - hc = Murmur3.hash128(offsetKeyBytes, 35, keyBytes.length, seed); - assertEquals(gl1, hc[0]); - assertEquals(gl2, hc[1]); - } - - @Test - public void testHashCodeM3_64() { - byte[] origin = ("It was the best of times, it was the worst of times," + - " it was the age of wisdom, it was the age of foolishness," + - " it was the epoch of belief, it was the epoch of incredulity," + - " it was the season of Light, it was the season of Darkness," + - " it was the spring of hope, it was the winter of despair," + - " we had everything before us, we had nothing before us," + - " we were all going direct to Heaven," + - " we were all going direct the other way.").getBytes(); - long hash = Murmur3.hash64(origin, 0, origin.length); - assertEquals(305830725663368540L, hash); - - byte[] originOffset = new byte[origin.length + 150]; - Arrays.fill(originOffset, (byte) 123); - System.arraycopy(origin, 0, originOffset, 150, origin.length); - hash = Murmur3.hash64(originOffset, 150, origin.length); - assertEquals(305830725663368540L, hash); - } - - @Test - public void testHashCodesM3_128_ints() { - int seed = 123; - Random rand = new Random(seed); - HashFunction hf = Hashing.murmur3_128(seed); - for (int i = 0; i < 1000; i++) { - int val = rand.nextInt(); - byte[] data = ByteBuffer.allocate(4).putInt(val).array(); - // guava stores the hashcodes in little endian order - ByteBuffer buf = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN); - buf.put(hf.hashBytes(data).asBytes()); - buf.flip(); - long gl1 = buf.getLong(); - long gl2 = buf.getLong(8); - long[] hc = Murmur3.hash128(data, 0, data.length, seed); - long m1 = hc[0]; - long m2 = hc[1]; - assertEquals(gl1, m1); - assertEquals(gl2, m2); - - byte[] offsetData = new byte[data.length + 50]; - System.arraycopy(data, 0, offsetData, 50, data.length); - hc = Murmur3.hash128(offsetData, 50, data.length, seed); - assertEquals(gl1, hc[0]); - assertEquals(gl2, hc[1]); - } - } - - @Test - public void testHashCodesM3_128_longs() { - int seed = 123; - Random rand = new Random(seed); - HashFunction hf = Hashing.murmur3_128(seed); - for (int i = 0; i < 1000; i++) { - long val = rand.nextLong(); - byte[] data = ByteBuffer.allocate(8).putLong(val).array(); - // guava stores the hashcodes in little endian order - ByteBuffer buf = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN); - buf.put(hf.hashBytes(data).asBytes()); - buf.flip(); - long gl1 = buf.getLong(); - long gl2 = buf.getLong(8); - long[] hc = Murmur3.hash128(data, 0, data.length, seed); - long m1 = hc[0]; - long m2 = hc[1]; - assertEquals(gl1, m1); - assertEquals(gl2, m2); - } - } - - @Test - public void testHashCodesM3_128_double() { - int seed = 123; - Random rand = new Random(seed); - HashFunction hf = Hashing.murmur3_128(seed); - for (int i = 0; i < 1000; i++) { - double val = rand.nextDouble(); - byte[] data = ByteBuffer.allocate(8).putDouble(val).array(); - // guava stores the hashcodes in little endian order - ByteBuffer buf = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN); - buf.put(hf.hashBytes(data).asBytes()); - buf.flip(); - long gl1 = buf.getLong(); - long gl2 = buf.getLong(8); - long[] hc = Murmur3.hash128(data, 0, data.length, seed); - long m1 = hc[0]; - long m2 = hc[1]; - assertEquals(gl1, m1); - assertEquals(gl2, m2); - } - } - - - @Test - public void testIncremental() { - final int seed = 123, arraySize = 1023; - byte[] bytes = new byte[arraySize]; - new Random(seed).nextBytes(bytes); - int expected = Murmur3.hash32(bytes); - Murmur3.IncrementalHash32 same = new IncrementalHash32(), diff = new IncrementalHash32(); - for (int blockSize = 1; blockSize <= arraySize; ++blockSize) { - byte[] block = new byte[blockSize]; - same.start(Murmur3.DEFAULT_SEED); - diff.start(Murmur3.DEFAULT_SEED); - for (int offset = 0; offset < arraySize; offset += blockSize) { - int length = Math.min(arraySize - offset, blockSize); - same.add(bytes, offset, length); - System.arraycopy(bytes, offset, block, 0, length); - diff.add(block, 0, length); - } - assertEquals("Block size " + blockSize, expected, same.end()); - assertEquals("Block size " + blockSize, expected, diff.end()); - } - } -}