diff --git a/common/pom.xml b/common/pom.xml
index aaeecc09556a76ced013a48ccfbdd7e782e6196b..67059462fa0ee12d6652a30f21846f4f70289dba 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -48,6 +48,11 @@
org.apache.hivehive-storage-api
+
+ org.apache.hive
+ hive-serde-api
+ ${serde-api.version}
+ commons-cli
diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java
deleted file mode 100644
index 53684e7ab1b8d473ac7618ab2c830c226175a18c..0000000000000000000000000000000000000000
--- a/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common.type;
-
-import org.apache.commons.lang.StringUtils;
-
-public abstract class HiveBaseChar {
- protected String value;
-
- protected HiveBaseChar() {
- }
-
- /**
- * Sets the string value to a new value, obeying the max length defined for this object.
- * @param val new value
- */
- public void setValue(String val, int maxLength) {
- value = HiveBaseChar.enforceMaxLength(val, maxLength);
- }
-
- public void setValue(HiveBaseChar val, int maxLength) {
- setValue(val.value, maxLength);
- }
-
- public static String enforceMaxLength(String val, int maxLength) {
- if (val == null) {
- return null;
- }
- String value = val;
-
- if (maxLength > 0) {
- int valLength = val.codePointCount(0, val.length());
- if (valLength > maxLength) {
- // Truncate the excess chars to fit the character length.
- // Also make sure we take supplementary chars into account.
- value = val.substring(0, val.offsetByCodePoints(0, maxLength));
- }
- }
- return value;
- }
-
- public static String getPaddedValue(String val, int maxLength) {
- if (val == null) {
- return null;
- }
- if (maxLength < 0) {
- return val;
- }
-
- int valLength = val.codePointCount(0, val.length());
- if (valLength > maxLength) {
- return enforceMaxLength(val, maxLength);
- }
-
- if (maxLength > valLength) {
- // Make sure we pad the right amount of spaces; valLength is in terms of code points,
- // while StringUtils.rpad() is based on the number of java chars.
- int padLength = val.length() + (maxLength - valLength);
- val = StringUtils.rightPad(val, padLength);
- }
- return val;
- }
-
- public String getValue() {
- return value;
- }
-
- public int getCharacterLength() {
- return value.codePointCount(0, value.length());
- }
-
- @Override
- public int hashCode() {
- return getValue().hashCode();
- }
-
- @Override
- public String toString() {
- return getValue();
- }
-}
diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java
deleted file mode 100644
index 66aa524bb2c3b0835cdb8af3001ec6d96b5ae67c..0000000000000000000000000000000000000000
--- a/common/src/java/org/apache/hadoop/hive/common/type/HiveChar.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common.type;
-
-import org.apache.commons.lang.StringUtils;
-
-/**
- * HiveChar.
- * String values will be padded to full char length.
- * Character legnth, comparison, hashCode should ignore trailing spaces.
- */
-public class HiveChar extends HiveBaseChar
- implements Comparable {
-
- public static final int MAX_CHAR_LENGTH = 255;
-
- public HiveChar() {
- }
-
- public HiveChar(String val, int len) {
- setValue(val, len);
- }
-
- public HiveChar(HiveChar hc, int len) {
- setValue(hc.value, len);
- }
-
- /**
- * Set char value, padding or truncating the value to the size of len parameter.
- */
- public void setValue(String val, int len) {
- super.setValue(HiveBaseChar.getPaddedValue(val, len), -1);
- }
-
- public void setValue(String val) {
- setValue(val, -1);
- }
-
- public String getStrippedValue() {
- return StringUtils.stripEnd(value, " ");
- }
-
- public String getPaddedValue() {
- return value;
- }
-
- public int getCharacterLength() {
- String strippedValue = getStrippedValue();
- return strippedValue.codePointCount(0, strippedValue.length());
- }
-
- public String toString() {
- return getPaddedValue();
- }
-
- public int compareTo(HiveChar rhs) {
- if (rhs == this) {
- return 0;
- }
- return this.getStrippedValue().compareTo(rhs.getStrippedValue());
- }
-
- public boolean equals(Object rhs) {
- if (rhs == this) {
- return true;
- }
- if (rhs == null || rhs.getClass() != getClass()) {
- return false;
- }
- return this.getStrippedValue().equals(((HiveChar) rhs).getStrippedValue());
- }
-
- public int hashCode() {
- return getStrippedValue().hashCode();
- }
-}
diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java
deleted file mode 100644
index 09009eb622ca938a718dcd8f7dc5343169c736c8..0000000000000000000000000000000000000000
--- a/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common.type;
-
-/**
- *
- * HiveVarChar.
- * String wrapper to support SQL VARCHAR features.
- * Max string length is enforced.
- *
- */
-public class HiveVarchar extends HiveBaseChar
- implements Comparable {
-
- public static final int MAX_VARCHAR_LENGTH = 65535;
-
- public HiveVarchar() {
- }
-
- public HiveVarchar(String val, int len) {
- setValue(val, len);
- }
-
- public HiveVarchar(HiveVarchar hc, int len) {
- setValue(hc, len);
- }
-
- /**
- * Set the new value
- */
- public void setValue(String val) {
- super.setValue(val, -1);
- }
-
- public void setValue(HiveVarchar hc) {
- super.setValue(hc.getValue(), -1);
- }
-
- public int compareTo(HiveVarchar rhs) {
- if (rhs == this) {
- return 0;
- }
- return this.getValue().compareTo(rhs.getValue());
- }
-
- public boolean equals(Object rhs) {
- if (rhs == this) {
- return true;
- }
- return this.getValue().equals(((HiveVarchar)rhs).getValue());
- }
-}
diff --git a/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZUtil.java b/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZUtil.java
index c49aefd5c201e37ee519be8dec1dd889d818d74f..90556aa88eee021a85d7e24ecaa4925b7f1af050 100644
--- a/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZUtil.java
+++ b/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZUtil.java
@@ -133,18 +133,4 @@ public static TimestampTZ convert(Date date, ZoneId defaultTimeZone) {
return parse(s, defaultTimeZone);
}
- public static ZoneId parseTimeZone(String timeZoneStr) {
- if (timeZoneStr == null || timeZoneStr.trim().isEmpty() ||
- timeZoneStr.trim().toLowerCase().equals("local")) {
- // default
- return ZoneId.systemDefault();
- }
- try {
- return ZoneId.of(timeZoneStr);
- } catch (DateTimeException e1) {
- // default
- throw new RuntimeException("Invalid time zone displacement value");
- }
- }
-
}
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 711dfbdc1f05a25ffc64d297c6b1b25853d99a57..1fcb17ca48db10a2269bd4095f9942b86a658083 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -26,7 +26,6 @@
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
-import org.apache.hadoop.hive.common.type.TimestampTZUtil;
import org.apache.hadoop.hive.conf.Validator.PatternSet;
import org.apache.hadoop.hive.conf.Validator.RangeValidator;
import org.apache.hadoop.hive.conf.Validator.RatioValidator;
@@ -34,6 +33,7 @@
import org.apache.hadoop.hive.conf.Validator.StringSet;
import org.apache.hadoop.hive.conf.Validator.TimeValidator;
import org.apache.hadoop.hive.conf.Validator.WritableDirectoryValidator;
+import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat;
@@ -4482,7 +4482,7 @@ private static String getSQLStdAuthDefaultWhiteListPattern() {
*/
public ZoneId getLocalTimeZone() {
String timeZoneStr = getVar(ConfVars.HIVE_LOCAL_TIME_ZONE);
- return TimestampTZUtil.parseTimeZone(timeZoneStr);
+ return TimestampLocalTZTypeInfo.parseTimeZone(timeZoneStr);
}
/**
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java
index a66c13507abef42977dfdb315ff7d69404f67ac3..3df02f3ea7e298e185d57d8a19ee5bcf1065dcca 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java
@@ -207,7 +207,7 @@ private static String determineFieldComment(String comment) {
public static FieldSchema getFieldSchemaFromTypeInfo(String fieldName,
TypeInfo typeInfo) {
return new FieldSchema(fieldName, typeInfo.getTypeName(),
- "generated by TypeInfoUtils.getFieldSchemaFromTypeInfo");
+ "generated by StorageSchemaUtils.getFieldSchemaFromTypeInfo");
}
}
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/SerDeStorageSchemaReader.java b/metastore/src/java/org/apache/hadoop/hive/metastore/SerDeStorageSchemaReader.java
deleted file mode 100644
index 59bcd5ca34d5083d357d7157abf3682399060a1a..0000000000000000000000000000000000000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/SerDeStorageSchemaReader.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.serde2.Deserializer;
-import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.utils.StringUtils;
-
-import java.util.List;
-
-public class SerDeStorageSchemaReader implements StorageSchemaReader {
- @Override
- public List readSchema(Table tbl, EnvironmentContext envContext, Configuration conf)
- throws MetaException {
- ClassLoader orgHiveLoader = null;
- try {
- if (envContext != null) {
- String addedJars = envContext.getProperties().get("hive.added.jars.path");
- if (org.apache.commons.lang.StringUtils.isNotBlank(addedJars)) {
- //for thread safe
- orgHiveLoader = conf.getClassLoader();
- ClassLoader loader = org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.addToClassPath(
- orgHiveLoader, org.apache.commons.lang.StringUtils.split(addedJars, ","));
- conf.setClassLoader(loader);
- }
- }
-
- Deserializer s = HiveMetaStoreUtils.getDeserializer(conf, tbl, false);
- return HiveMetaStoreUtils.getFieldsFromDeserializer(tbl.getTableName(), s);
- } catch (Exception e) {
- StringUtils.stringifyException(e);
- throw new MetaException(e.getMessage());
- } finally {
- if (orgHiveLoader != null) {
- conf.setClassLoader(orgHiveLoader);
- }
- }
- }
-}
diff --git a/pom.xml b/pom.xml
index 6d1667585216ca2d329bc9d62901aac97caaba45..b9c265fa9bd113c7cd254fb94f156964261e217d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -194,6 +194,7 @@
1.7.104.0.43.0.0-SNAPSHOT
+ 3.0.0-SNAPSHOT0.9.1-SNAPSHOT0.92.0-incubating2.2.0
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java
index 1e044b4dcd5bc6dc03cb5757fc53d4b468771172..02bb82f2e74a1cba074c5a42f4be95cfc9d2e701 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java
@@ -26,7 +26,7 @@
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveTypeEntry;
/**
* common class for reflective UDFs
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java
index 05e21633ad34b744d2844b7a7fa4b308a2d3e55e..70243c09281888b9482ea9006d7b612f7c5cb7ba 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java
@@ -39,7 +39,7 @@
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveTypeEntry;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.BytesWritable;
@@ -105,9 +105,9 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen
return returnOI;
}
- private PrimitiveObjectInspectorUtils.PrimitiveTypeEntry getTypeFor(Class> retType)
+ private PrimitiveTypeEntry getTypeFor(Class> retType)
throws UDFArgumentException {
- PrimitiveObjectInspectorUtils.PrimitiveTypeEntry entry =
+ PrimitiveTypeEntry entry =
PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaType(retType);
if (entry == null) {
entry = PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaClass(retType);
diff --git a/serde/pom.xml b/serde/pom.xml
index 0247c32452180aad73eb1932110096c1d044bd15..3ec09a2b028284fad9565e4f113487fef51eee5a 100644
--- a/serde/pom.xml
+++ b/serde/pom.xml
@@ -49,6 +49,11 @@
hive-shims${project.version}
+
+ org.apache.hive
+ hive-serde-api
+ ${storage-api.version}
+ com.google.code.findbugs
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java
deleted file mode 100644
index a2a85b3dc44d7d7d676ad17eb476edbb5560f1cc..0000000000000000000000000000000000000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.serde2;
-
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.io.Writable;
-
-import javax.annotation.Nullable;
-
-/**
- * Abstract class for implementing SerDe. The abstract class has been created, so that
- * new methods can be added in the underlying interface, SerDe, and only implementations
- * that need those methods overwrite it.
- */
-public abstract class AbstractSerDe implements Deserializer, Serializer {
-
- protected String configErrors;
-
- /**
- * Initialize the SerDe. By default, this will use one set of properties, either the
- * table properties or the partition properties. If a SerDe needs access to both sets,
- * it should override this method.
- *
- * Eventually, once all SerDes have implemented this method,
- * we should convert it to an abstract method.
- *
- * @param configuration Hadoop configuration
- * @param tableProperties Table properties
- * @param partitionProperties Partition properties
- * @throws SerDeException
- */
- public void initialize(Configuration configuration, Properties tableProperties,
- Properties partitionProperties) throws SerDeException {
- initialize(configuration,
- SerDeUtils.createOverlayedProperties(tableProperties, partitionProperties));
- }
-
- /**
- * Initialize the HiveSerializer.
- *
- * @param conf
- * System properties. Can be null in compile time
- * @param tbl
- * table properties
- * @throws SerDeException
- */
- @Deprecated
- public abstract void initialize(@Nullable Configuration conf, Properties tbl)
- throws SerDeException;
-
- /**
- * Returns the Writable class that would be returned by the serialize method.
- * This is used to initialize SequenceFile header.
- */
- public abstract Class extends Writable> getSerializedClass();
-
- /**
- * Serialize an object by navigating inside the Object with the
- * ObjectInspector. In most cases, the return value of this function will be
- * constant since the function will reuse the Writable object. If the client
- * wants to keep a copy of the Writable, the client needs to clone the
- * returned value.
- */
- public abstract Writable serialize(Object obj, ObjectInspector objInspector)
- throws SerDeException;
-
- /**
- * Returns statistics collected when serializing
- */
- public abstract SerDeStats getSerDeStats();
-
- /**
- * Deserialize an object out of a Writable blob. In most cases, the return
- * value of this function will be constant since the function will reuse the
- * returned object. If the client wants to keep a copy of the object, the
- * client needs to clone the returned value by calling
- * ObjectInspectorUtils.getStandardObject().
- *
- * @param blob
- * The Writable object containing a serialized object
- * @return A Java object representing the contents in the blob.
- */
- public abstract Object deserialize(Writable blob) throws SerDeException;
-
- /**
- * Get the object inspector that can be used to navigate through the internal
- * structure of the Object returned from deserialize(...).
- */
- public abstract ObjectInspector getObjectInspector() throws SerDeException;
-
- /**
- * Get the error messages during the Serde configuration
- *
- * @return The error messages in the configuration which are empty if no error occurred
- */
- public String getConfigurationErrors() {
- return configErrors == null ? "" : configErrors;
- }
-
- /**
- * @return Whether the SerDe that can store schema both inside and outside of metastore
- * does, in fact, store it inside metastore, based on table parameters.
- */
- public boolean shouldStoreFieldsInMetastore(Map tableParams) {
- return false; // The default, unless SerDe overrides it.
- }
-}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/Deserializer.java b/serde/src/java/org/apache/hadoop/hive/serde2/Deserializer.java
deleted file mode 100644
index a1d3dd87665e2358209d92a4f9efcbe5c57418ea..0000000000000000000000000000000000000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/Deserializer.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.serde2;
-
-import java.util.Properties;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.io.Writable;
-
-/**
- * HiveDeserializer is used to deserialize the data from hadoop Writable to a
- * custom java object that can be of any type that the developer wants.
- *
- * HiveDeserializer also provides the ObjectInspector which can be used to
- * inspect the internal structure of the object (that is returned by deserialize
- * function).
- * All deserializers should extend the abstract class AbstractDeserializer.
- * The interface is necessary for SerDes to be able to implement both Serializer and Deserializer.
- */
-public interface Deserializer {
-
- /**
- * Initialize the HiveDeserializer.
- *
- * @param conf
- * System properties
- * @param tbl
- * table properties
- * @throws SerDeException
- */
- void initialize(Configuration conf, Properties tbl) throws SerDeException;
-
- /**
- * Deserialize an object out of a Writable blob. In most cases, the return
- * value of this function will be constant since the function will reuse the
- * returned object. If the client wants to keep a copy of the object, the
- * client needs to clone the returned deserialized value by calling
- * ObjectInspectorUtils.getStandardObject().
- *
- * @param blob
- * The Writable object containing a serialized object
- * @return A Java object representing the contents in the blob.
- */
- Object deserialize(Writable blob) throws SerDeException;
-
- /**
- * Get the object inspector that can be used to navigate through the internal
- * structure of the Object returned from deserialize(...).
- */
- ObjectInspector getObjectInspector() throws SerDeException;
-
- /**
- * Returns statistics collected when serializing
- */
- SerDeStats getSerDeStats();
-}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeException.java b/serde/src/java/org/apache/hadoop/hive/serde2/SerDeException.java
deleted file mode 100644
index ea1ae9ca006754a19c0ab51d4d02be125476ee7e..0000000000000000000000000000000000000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeException.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.serde2;
-
-/**
- * Generic exception class for SerDes.
- *
- */
-
-public class SerDeException extends Exception {
- private static final long serialVersionUID = 1L;
-
- public SerDeException() {
- super();
- }
-
- public SerDeException(String message) {
- super(message);
- }
-
- public SerDeException(Throwable cause) {
- super(cause);
- }
-
- public SerDeException(String message, Throwable cause) {
- super(message, cause);
- }
-}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeStats.java b/serde/src/java/org/apache/hadoop/hive/serde2/SerDeStats.java
deleted file mode 100644
index 6cf2ccdcffff06bae21986c3df4c29fad8b560c5..0000000000000000000000000000000000000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeStats.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.serde2;
-
-public class SerDeStats {
-
- /**
- * Class used to pass statistics information from serializer/deserializer to the tasks.
- * A SerDeStats object is returned by calling SerDe.getStats().
- */
-
- // currently we support only raw data size stat
- private long rawDataSize;
- private long rowCount;
-
- public SerDeStats() {
- rawDataSize = 0;
- rowCount = 0;
- }
-
- /**
- * Return the raw data size
- * @return raw data size
- */
- public long getRawDataSize() {
- return rawDataSize;
- }
-
- /**
- * Set the raw data size
- * @param uSize - size to be set
- */
- public void setRawDataSize(long uSize) {
- rawDataSize = uSize;
- }
-
- /**
- * Return the row count
- * @return row count
- */
- public long getRowCount() {
- return rowCount;
- }
-
- /**
- * Set the row count
- * @param rowCount - count of rows
- */
- public void setRowCount(long rowCount) {
- this.rowCount = rowCount;
- }
-
-}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/Serializer.java b/serde/src/java/org/apache/hadoop/hive/serde2/Serializer.java
deleted file mode 100644
index 3f07a86e31cdcb1ee082cda42163a192362d14ea..0000000000000000000000000000000000000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/Serializer.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.serde2;
-
-import java.util.Properties;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.io.Writable;
-
-/**
- * HiveSerializer is used to serialize data to a Hadoop Writable object. The
- * serialize In addition to the interface below, all implementations are assume
- * to have a ctor that takes a single 'Table' object as argument.
- * All serializers should extend the abstract class AbstractSerializer.
- * The interface is necessary for SerDes to be able to implement both Serializer and Deserializer.
- */
-public interface Serializer {
-
- /**
- * Initialize the HiveSerializer.
- *
- * @param conf
- * System properties
- * @param tbl
- * table properties
- * @throws SerDeException
- */
- void initialize(Configuration conf, Properties tbl) throws SerDeException;
-
- /**
- * Returns the Writable class that would be returned by the serialize method.
- * This is used to initialize SequenceFile header.
- */
- Class extends Writable> getSerializedClass();
-
- /**
- * Serialize an object by navigating inside the Object with the
- * ObjectInspector. In most cases, the return value of this function will be
- * constant since the function will reuse the Writable object. If the client
- * wants to keep a copy of the Writable, the client needs to clone the
- * returned value.
- */
- Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException;
-
- /**
- * Returns statistics collected when serializing
- */
- SerDeStats getSerDeStats();
-}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java
index 35d83bdb1af03df674842b72422e4b7d4d22b596..1998b94b26c3248d9806ff3646d4f50d38661c03 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java
@@ -36,7 +36,7 @@
import java.util.Set;
import org.apache.avro.Schema;
-import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfoValidationUtils;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -143,7 +143,7 @@ public static TypeInfo generateTypeInfo(Schema schema,
}
try {
- HiveDecimalUtils.validateParameter(precision, scale);
+ PrimitiveTypeInfoValidationUtils.validateParameter(precision, scale);
} catch (Exception ex) {
throw new AvroSerdeException("Invalid precision or scale for decimal type", ex);
}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java
index 0c8a4664e7e9454b5492ac9e3067c5211a49e64c..37675463054a371d613bd86de376859a0731ed93 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java
@@ -36,7 +36,7 @@
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveTypeEntry;
import org.apache.hadoop.hive.serde2.thrift.ConfigurableTProtocol;
import org.apache.hadoop.hive.serde2.thrift.TReflectionUtils;
import org.apache.hadoop.io.BytesWritable;
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveBaseCharWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveBaseCharWritable.java
deleted file mode 100644
index 8c37a9bb75f2e425099461fd28397f1dceed97b4..0000000000000000000000000000000000000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveBaseCharWritable.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.serde2.io;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hive.common.util.HiveStringUtils;
-
-public abstract class HiveBaseCharWritable {
- protected Text value = new Text();
-
- public HiveBaseCharWritable() {
- }
-
- public int getCharacterLength() {
- return HiveStringUtils.getTextUtfLength(value);
- }
-
- /**
- * Access to the internal Text member. Use with care.
- * @return
- */
- public Text getTextValue() {
- return value;
- }
-
- public void readFields(DataInput in) throws IOException {
- value.readFields(in);
- }
-
- public void write(DataOutput out) throws IOException {
- value.write(out);
- }
-
- public boolean equals(Object obj) {
- if (obj == null || (obj.getClass() != this.getClass())) {
- return false;
- }
- return value.equals(((HiveBaseCharWritable)obj).value);
- }
-
- public int hashCode() {
- return value.hashCode();
- }
-}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveCharWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveCharWritable.java
deleted file mode 100644
index 2236a8799980fa1ba925b7b2afb5e4b0dbaf073e..0000000000000000000000000000000000000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveCharWritable.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.serde2.io;
-
-import org.apache.hadoop.hive.common.type.HiveBaseChar;
-import org.apache.hadoop.hive.common.type.HiveChar;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hive.common.util.HiveStringUtils;
-
-/**
- * HiveCharWritable.
- * String values will be padded to full char length.
- * Character legnth, comparison, hashCode should ignore trailing spaces.
- */
-public class HiveCharWritable extends HiveBaseCharWritable
- implements WritableComparable {
-
- public HiveCharWritable() {
- }
-
- public HiveCharWritable(HiveChar hc) {
- set(hc);
- }
-
- public HiveCharWritable(HiveCharWritable hcw) {
- set(hcw);
- }
-
- public void set(HiveChar val) {
- set(val.getValue(), -1);
- }
-
- public void set(String val) {
- set(val, -1);
- }
-
- public void set(HiveCharWritable val) {
- value.set(val.value);
- }
-
- public void set(HiveCharWritable val, int maxLength) {
- set(val.getHiveChar(), maxLength);
- }
-
- public void set(HiveChar val, int len) {
- set(val.getValue(), len);
- }
-
- public void set(String val, int maxLength) {
- value.set(HiveBaseChar.getPaddedValue(val, maxLength));
- }
-
- public HiveChar getHiveChar() {
- // Copy string value as-is
- return new HiveChar(value.toString(), -1);
- }
-
- public void enforceMaxLength(int maxLength) {
- if (getCharacterLength()!=maxLength)
- set(getHiveChar(), maxLength);
- }
-
- public Text getStrippedValue() {
- // A lot of these methods could be done more efficiently by operating on the Text value
- // directly, rather than converting to HiveChar.
- return new Text(getHiveChar().getStrippedValue());
- }
-
- public Text getPaddedValue() {
- return getTextValue();
- }
-
- public int getCharacterLength() {
- return HiveStringUtils.getTextUtfLength(getStrippedValue());
- }
-
- public int compareTo(HiveCharWritable rhs) {
- return getStrippedValue().compareTo(rhs.getStrippedValue());
- }
-
- public boolean equals(Object rhs) {
- if (rhs == this) {
- return true;
- }
- if (rhs == null || rhs.getClass() != getClass()) {
- return false;
- }
- return this.getStrippedValue().equals(((HiveCharWritable) rhs).getStrippedValue());
- }
-
- public int hashCode() {
- return getStrippedValue().hashCode();
- }
-
- @Override
- public String toString() {
- return getPaddedValue().toString();
- }
-}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java
deleted file mode 100644
index 270d97fa6a856916f95006f0a29506dc9391d9aa..0000000000000000000000000000000000000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.serde2.io;
-
-import org.apache.hadoop.hive.common.type.HiveBaseChar;
-import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hadoop.io.WritableComparable;
-
-public class HiveVarcharWritable extends HiveBaseCharWritable
- implements WritableComparable{
-
- public HiveVarcharWritable() {
- }
-
- public HiveVarcharWritable(HiveVarchar hc) {
- set(hc);
- }
-
- public HiveVarcharWritable(HiveVarcharWritable hcw) {
- set(hcw);
- }
-
- public void set(HiveVarchar val) {
- set(val.getValue());
- }
-
- public void set(String val) {
- set(val, -1); // copy entire string value
- }
-
- public void set(HiveVarcharWritable val) {
- value.set(val.value);
- }
-
- public void set(HiveVarcharWritable val, int maxLength) {
- set(val.getHiveVarchar(), maxLength);
- }
-
- public void set(HiveVarchar val, int len) {
- set(val.getValue(), len);
- }
-
- public void set(String val, int maxLength) {
- value.set(HiveBaseChar.enforceMaxLength(val, maxLength));
- }
-
- public HiveVarchar getHiveVarchar() {
- return new HiveVarchar(value.toString(), -1);
- }
-
- public void enforceMaxLength(int maxLength) {
- if (getCharacterLength() > maxLength) {
- set(value.toString(), maxLength);
- }
- }
-
- @Override
- public int compareTo(HiveVarcharWritable rhs) {
- return value.compareTo(rhs.value);
- }
-
- @Override
- public String toString() {
- return value.toString();
- }
-}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ConstantObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ConstantObjectInspector.java
deleted file mode 100644
index 1adc72b0897fa39f8144bb002ce17d5e47f0cce4..0000000000000000000000000000000000000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ConstantObjectInspector.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.serde2.objectinspector;
-
-import org.apache.hadoop.hive.common.classification.InterfaceAudience;
-import org.apache.hadoop.hive.common.classification.InterfaceStability;
-
-/**
- * ConstantObjectInspector. This interface should be implemented by
- * ObjectInspectors which represent constant values and can return them without
- * an evaluation.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public interface ConstantObjectInspector extends ObjectInspector {
-
- Object getWritableConstantValue();
-
-}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectInspector.java
deleted file mode 100644
index 56d59e7089b15543f114c1a041b36644386fa776..0000000000000000000000000000000000000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectInspector.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.serde2.objectinspector;
-
-import org.apache.hadoop.hive.common.classification.InterfaceAudience;
-import org.apache.hadoop.hive.common.classification.InterfaceStability;
-
-import java.util.List;
-
-/**
- * ListObjectInspector.
- *
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public interface ListObjectInspector extends ObjectInspector {
-
- // ** Methods that does not need a data object **
- ObjectInspector getListElementObjectInspector();
-
- // ** Methods that need a data object **
- /**
- * returns null for null list, out-of-the-range index.
- */
- Object getListElement(Object data, int index);
-
- /**
- * returns -1 for data = null.
- */
- int getListLength(Object data);
-
- /**
- * returns null for data = null.
- *
- * Note: This method should not return a List object that is reused by the
- * same ListObjectInspector, because it's possible that the same
- * ListObjectInspector will be used in multiple places in the code.
- *
- * However it's OK if the List object is part of the Object data.
- */
- List> getList(Object data);
-
-}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapObjectInspector.java
deleted file mode 100644
index 40b11f05f76db7dfbe2a6e4a17320aefbd7c374f..0000000000000000000000000000000000000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapObjectInspector.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.serde2.objectinspector;
-
-import org.apache.hadoop.hive.common.classification.InterfaceAudience;
-import org.apache.hadoop.hive.common.classification.InterfaceStability;
-
-import java.util.Map;
-
-/**
- * MapObjectInspector.
- *
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public interface MapObjectInspector extends ObjectInspector {
-
- // ** Methods that does not need a data object **
- // Map Type
- ObjectInspector getMapKeyObjectInspector();
-
- ObjectInspector getMapValueObjectInspector();
-
- // ** Methods that need a data object **
- // In this function, key has to be of the same structure as the Map expects.
- // Most cases key will be primitive type, so it's OK.
- // In rare cases that key is not primitive, the user is responsible for
- // defining
- // the hashCode() and equals() methods of the key class.
- Object getMapValueElement(Object data, Object key);
-
- /**
- * returns null for data = null.
- *
- * Note: This method should not return a Map object that is reused by the same
- * MapObjectInspector, because it's possible that the same MapObjectInspector
- * will be used in multiple places in the code.
- *
- * However it's OK if the Map object is part of the Object data.
- */
- Map, ?> getMap(Object data);
-
- /**
- * returns -1 for NULL map.
- */
- int getMapSize(Object data);
-}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java
deleted file mode 100644
index 5ed429de737e33154f66b106a7a53dafbd179071..0000000000000000000000000000000000000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.serde2.objectinspector;
-
-import org.apache.hadoop.hive.common.classification.InterfaceAudience;
-import org.apache.hadoop.hive.common.classification.InterfaceStability;
-
-/**
- * ObjectInspector helps us to look into the internal structure of a complex
- * object.
- *
- * A (probably configured) ObjectInspector instance stands for a specific type
- * and a specific way to store the data of that type in the memory.
- *
- * For native java Object, we can directly access the internal structure through
- * member fields and methods. ObjectInspector is a way to delegate that
- * functionality away from the Object, so that we have more control on the
- * behavior of those actions.
- *
- * An efficient implementation of ObjectInspector should rely on factory, so
- * that we can make sure the same ObjectInspector only has one instance. That
- * also makes sure hashCode() and equals() methods of java.lang.Object directly
- * works for ObjectInspector as well.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public interface ObjectInspector extends Cloneable {
-
- /**
- * Category.
- *
- */
- public static enum Category {
- PRIMITIVE, LIST, MAP, STRUCT, UNION
- };
-
- /**
- * Returns the name of the data type that is inspected by this
- * ObjectInspector. This is used to display the type information to the user.
- *
- * For primitive types, the type name is standardized. For other types, the
- * type name can be something like "list<int>", "map<int,string>", java class
- * names, or user-defined type names similar to typedef.
- */
- String getTypeName();
-
- /**
- * An ObjectInspector must inherit from one of the following interfaces if
- * getCategory() returns: PRIMITIVE: PrimitiveObjectInspector LIST:
- * ListObjectInspector MAP: MapObjectInspector STRUCT: StructObjectInspector.
- */
- Category getCategory();
-}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java
deleted file mode 100644
index 3c58f066f2c5c9748afe220b71867a0d4fc68470..0000000000000000000000000000000000000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.serde2.objectinspector;
-
-import org.apache.hadoop.hive.common.classification.InterfaceAudience;
-import org.apache.hadoop.hive.common.classification.InterfaceStability;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-
-
-/**
- * PrimitiveObjectInspector.
- *
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public interface PrimitiveObjectInspector extends ObjectInspector {
-
- /**
- * The primitive types supported by Hive.
- */
- public static enum PrimitiveCategory {
- VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING,
- DATE, TIMESTAMP, TIMESTAMPLOCALTZ, BINARY, DECIMAL, VARCHAR, CHAR,
- INTERVAL_YEAR_MONTH, INTERVAL_DAY_TIME, UNKNOWN
- };
-
- public PrimitiveTypeInfo getTypeInfo();
-
- /**
- * Get the primitive category of the PrimitiveObjectInspector.
- */
- PrimitiveCategory getPrimitiveCategory();
-
- /**
- * Get the Primitive Writable class which is the return type of
- * getPrimitiveWritableObject() and copyToPrimitiveWritableObject().
- */
- Class> getPrimitiveWritableClass();
-
- /**
- * Return the data in an instance of primitive writable Object. If the Object
- * is already a primitive writable Object, just return o.
- */
- Object getPrimitiveWritableObject(Object o);
-
- /**
- * Get the Java Primitive class which is the return type of
- * getJavaPrimitiveObject().
- */
- Class> getJavaPrimitiveClass();
-
- /**
- * Get the Java Primitive object.
- */
- Object getPrimitiveJavaObject(Object o);
-
- /**
- * Get a copy of the Object in the same class, so the return value can be
- * stored independently of the parameter.
- *
- * If the Object is a Primitive Java Object, we just return the parameter
- * since Primitive Java Object is immutable.
- */
- Object copyObject(Object o);
-
- /**
- * Whether the ObjectInspector prefers to return a Primitive Writable Object
- * instead of a Primitive Java Object. This can be useful for determining the
- * most efficient way to getting data out of the Object.
- */
- boolean preferWritable();
-
- /**
- * The precision of the underlying data.
- */
- int precision();
-
- /**
- * The scale of the underlying data.
- */
- int scale();
-
-}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java
deleted file mode 100644
index dc147d6565725817cd7411f224ca93cd648a6f9c..0000000000000000000000000000000000000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.serde2.objectinspector;
-
-import org.apache.hadoop.hive.common.classification.InterfaceAudience;
-import org.apache.hadoop.hive.common.classification.InterfaceStability;
-
-/**
- * Classes implementing this interface are considered to represent a field of a
- * struct for this serde package.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public interface StructField {
-
- /**
- * Get the name of the field. The name should be always in lower-case.
- */
- String getFieldName();
-
- /**
- * Get the ObjectInspector for the field.
- */
- ObjectInspector getFieldObjectInspector();
-
- /**
- * Get the fieldID for the field.
- */
- int getFieldID();
-
- /**
- * Get the comment for the field. May be null if no comment provided.
- */
- String getFieldComment();
-}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructObjectInspector.java
deleted file mode 100644
index f7463543f35d876023a5426a95c9d256eff5660a..0000000000000000000000000000000000000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructObjectInspector.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.serde2.objectinspector;
-
-import org.apache.hadoop.hive.common.classification.InterfaceAudience;
-import org.apache.hadoop.hive.common.classification.InterfaceStability;
-
-import java.util.List;
-
-/**
- * StructObjectInspector.
- *
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public abstract class StructObjectInspector implements ObjectInspector {
-
- // ** Methods that does not need a data object **
- /**
- * Returns all the fields.
- */
- public abstract List extends StructField> getAllStructFieldRefs();
-
- /**
- * Look up a field.
- */
- public abstract StructField getStructFieldRef(String fieldName);
-
- // ** Methods that need a data object **
- /**
- * returns null for data = null.
- */
- public abstract Object getStructFieldData(Object data, StructField fieldRef);
-
- /**
- * returns null for data = null.
- */
- public abstract List
+ org.apache.hive
+ hive-storage-api
+ ${storage-api.version}
+
+ javax.servlet.jspjavax.servlet.jsp-api${javax-servlet-jsp.version}
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index 85c3222640f8bc78781d944bca66ec67472468c0..bdded94f2bb3562263e533a1d1a42f310d5ae623 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -229,13 +229,18 @@
-
+
org.apache.hivehive-storage-api${storage-api.version}
+ org.apache.hive
+ hive-serde-api
+ ${storage-api.version}
+
+ org.apache.logging.log4jlog4j-slf4j-impl${log4j2.version}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java
index d5dea4dc3ca6a83a863326e1c75e2480898d00af..124d6508ce0a44661d1fe7abc7206ee129b617fa 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java
@@ -224,7 +224,7 @@ public static boolean areColTypesCompatible(String from, String to) {
if (from.equals(VOID_TYPE_NAME)) return true;
// Allow date to string casts. NOTE: I suspect this is the reverse of what we actually
- // want, but it matches the code in o.a.h.h.serde2.typeinfo.TypeInfoUtils. I can't see how
+ // want, but it matches the code in o.a.h.h.serde2.typeinfo.StorageSchemaUtils. I can't see how
// users would be altering date columns into string columns. The other I easily see since
// Hive did not originally support datetime types. Also, the comment in the Hive code
// says string to date, even though the code does the opposite. But for now I'm keeping
@@ -248,6 +248,8 @@ public static boolean areColTypesCompatible(String from, String to) {
public static final char COLUMN_COMMENTS_DELIMITER = '\0';
+ public static final String LIST_COLUMN_COMMENTS = "columns.comments";
+
private static HashMap typeToThriftTypeMap;
static {
typeToThriftTypeMap = new HashMap<>();
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultStorageSchemaReader.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultStorageSchemaReader.java
deleted file mode 100644
index 1dbfa4272cd5368242d335fbde564d35cac26bba..0000000000000000000000000000000000000000
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultStorageSchemaReader.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Table;
-
-import java.util.List;
-
-/**
- * Default StorageSchemaReader. This just throws as the metastore currently doesn't know how to
- * read schemas from storage.
- */
-public class DefaultStorageSchemaReader implements StorageSchemaReader {
- @Override
- public List readSchema(Table tbl, EnvironmentContext envContext,
- Configuration conf) throws MetaException {
- throw new UnsupportedOperationException("Storage schema reading not supported");
- }
-}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 69d26c4fee9fa0301d351c4aeee9e0b64a144b4e..2026a19e9ceca9e176dcd8043aec3caf429e277e 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -53,7 +53,6 @@
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
-import java.util.function.Consumer;
import java.util.regex.Pattern;
import javax.jdo.JDOException;
@@ -132,6 +131,7 @@
import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
import org.apache.hadoop.hive.metastore.metrics.PerfLogger;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
+import org.apache.hadoop.hive.metastore.schema.reader.StorageSchemaReader;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager;
import org.apache.hadoop.hive.metastore.security.TUGIContainingTransport;
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/StorageSchemaReader.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/StorageSchemaReader.java
deleted file mode 100644
index 6251e23991b8f898f939f5848fc5cbf5e8ceb07c..0000000000000000000000000000000000000000
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/StorageSchemaReader.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Table;
-
-import java.util.List;
-
-/**
- * An interface to implement reading schemas from stored data.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-interface StorageSchemaReader {
- /**
- * Read the schema from the storage representation of the table.
- * @param tbl metastore table object
- * @param envContext environment context
- * @param conf current configuration file
- * @return list of field schemas
- * @throws MetaException if the table storage could not be read
- */
- List readSchema(Table tbl, EnvironmentContext envContext, Configuration conf)
- throws MetaException;
-}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 57692d3e92ac820efa15fa54bea86e90dcfe5d93..0e4beb8236be3b2919d97614e07932b543910e46 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -19,7 +19,7 @@
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader;
+import org.apache.hadoop.hive.metastore.schema.reader.DefaultStorageSchemaReader;
import org.apache.hadoop.hive.metastore.HiveAlterHandler;
import org.apache.hadoop.hive.metastore.MetastoreTaskThread;
import org.apache.hadoop.hive.metastore.events.EventCleanerTask;
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/DefaultStorageSchemaReader.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/DefaultStorageSchemaReader.java
new file mode 100644
index 0000000000000000000000000000000000000000..31a7fcfa0c240bb1f448f911a6f1f9bc6c691cc4
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/DefaultStorageSchemaReader.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.schema.reader;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.ColumnType;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.schema.utils.AvroSchemaUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.schema.utils.StorageSchemaUtils;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.avro.AvroSerdeException;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+
+import static org.apache.hadoop.hive.metastore.ColumnType.LIST_COLUMN_COMMENTS;
+
+/**
+ * Default StorageSchemaReader. This just throws as the metastore currently doesn't know how to
+ * read schemas from storage.
+ */
+public class DefaultStorageSchemaReader implements StorageSchemaReader {
+ private final static Logger LOG = LoggerFactory.getLogger(DefaultStorageSchemaReader.class);
+
+ private static final String AVRO_SERIALIZATION_LIB =
+ "org.apache.hadoop.hive.serde2.avro.AvroSerDe";
+
+ @Override
+ public List readSchema(Table tbl, EnvironmentContext envContext,
+ Configuration conf) throws MetaException {
+ String serializationLib = tbl.getSd().getSerdeInfo().getSerializationLib();
+ if (null == serializationLib || MetastoreConf
+ .getStringCollection(conf, MetastoreConf.ConfVars.SERDES_USING_METASTORE_FOR_SCHEMA)
+ .contains(serializationLib)) {
+ //safety check to make sure we should be using storage schema reader for this table
+ throw new MetaException(
+ "Invalid usage of default storage schema reader for table " + tbl.getTableName()
+ + " with storage descriptor " + tbl.getSd().getSerdeInfo().getSerializationLib());
+ }
+ Properties tblMetadataProperties = MetaStoreUtils.getTableMetadata(tbl);
+ if(AVRO_SERIALIZATION_LIB.equals(serializationLib)) {
+ //in case of avro table use AvroStorageSchemaReader utils
+ try {
+ return AvroSchemaUtils.getFieldsFromAvroSchema(conf, tblMetadataProperties);
+ } catch (AvroSerdeException e) {
+ LOG.warn("Exception received while reading avro schema for table " + tbl.getTableName(), e);
+ throw new MetaException(e.getMessage());
+ } catch (IOException e) {
+ LOG.warn("Exception received while reading avro schema for table " + tbl.getTableName(), e);
+ throw new MetaException(e.getMessage());
+ }
+ } else {
+ return getFieldSchemasFromTableMetadata(tblMetadataProperties);
+ }
+ }
+
+ /**
+ * This method implements a generic way to get the FieldSchemas from the table metadata
+ * properties like column names and column types. Most of the serdes have the same implemention
+ * in their initialize method
+ * //TODO refactor the common code from the serdes and move it to serde-api so that there is no
+ * //duplicate code
+ *
+ * @return list of FieldSchema objects
+ */
+ public static List getFieldSchemasFromTableMetadata(
+ Properties tblMetadataProperties) {
+ List columnNames = null;
+ List columnTypes = null;
+ // Get column names and types
+ String columnNameProperty = tblMetadataProperties.getProperty(serdeConstants.LIST_COLUMNS);
+ String columnTypeProperty = tblMetadataProperties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
+ final String columnNameDelimiter = tblMetadataProperties
+ .containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? tblMetadataProperties
+ .getProperty(serdeConstants.COLUMN_NAME_DELIMITER) : String
+ .valueOf(StorageSchemaUtils.COMMA);
+ // all table column names
+ if (columnNameProperty.isEmpty()) {
+ columnNames = Collections.emptyList();
+ } else {
+ columnNames = Arrays.asList(columnNameProperty.split(columnNameDelimiter));
+ }
+
+ // all column types
+ if (columnTypeProperty.isEmpty()) {
+ columnTypes = Collections.emptyList();
+ } else {
+ columnTypes = StorageSchemaUtils.getTypeInfosFromTypeString(columnTypeProperty);
+ }
+
+ final String columnCommentProperty =
+ tblMetadataProperties.getProperty(LIST_COLUMN_COMMENTS, "");
+ List columnComments = null;
+ if (columnCommentProperty == null || columnCommentProperty.isEmpty()) {
+ columnComments = new ArrayList<>(0);
+ } else {
+ columnComments = Arrays.asList(
+ columnCommentProperty.split(String.valueOf(ColumnType.COLUMN_COMMENTS_DELIMITER)));
+ }
+ LOG.debug("columns: {}, {}", columnNameProperty, columnNames);
+ LOG.debug("types: {}, {} ", columnTypeProperty, columnTypes);
+ LOG.debug("comments: {} ", columnCommentProperty);
+ return getFieldSchemaFromColumnInfo(columnNames, columnTypes, columnComments);
+ }
+
+ private static List getFieldSchemaFromColumnInfo(List columnNames,
+ List columnTypes, List columnComments) {
+ int len = columnNames.size();
+ List fieldSchemas = new ArrayList<>(len);
+ for (int i = 0; i < len; i++) {
+ FieldSchema fieldSchema = new FieldSchema();
+ fieldSchema.setName(columnNames.get(i));
+ //In case of complex types getTypeName() will recusively go into typeName
+ //of individual fields when the ColumnType was constructed
+ //in SchemaToTypeInfo.generateColumnTypes in the constructor
+ fieldSchema.setType(columnTypes.get(i).getTypeName());
+ fieldSchema.setComment(StorageSchemaUtils.determineFieldComment(columnComments.get(i)));
+ }
+ return fieldSchemas;
+ }
+}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/SerDeStorageSchemaReader.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/SerDeStorageSchemaReader.java
new file mode 100644
index 0000000000000000000000000000000000000000..fd09dd49bf0b4bd13bbff6d5b2d478b503257fc4
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/SerDeStorageSchemaReader.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.schema.reader;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.schema.utils.SerDeUtils;
+import org.apache.hadoop.hive.metastore.utils.StringUtils;
+
+import java.util.List;
+
+/**
+ * In order to use this Storage schema reader you should add the hive-serde jar in the classpath
+ * of the metastore.
+ */
+public class SerDeStorageSchemaReader implements StorageSchemaReader {
+ @Override
+ public List readSchema(Table tbl, EnvironmentContext envContext, Configuration conf)
+ throws MetaException {
+ ClassLoader orgHiveLoader = null;
+ try {
+ if (envContext != null) {
+ String addedJars = envContext.getProperties().get("hive.added.jars.path");
+ if (org.apache.commons.lang.StringUtils.isNotBlank(addedJars)) {
+ //for thread safe
+ orgHiveLoader = conf.getClassLoader();
+ ClassLoader loader = org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.addToClassPath(
+ orgHiveLoader, org.apache.commons.lang.StringUtils.split(addedJars, ","));
+ conf.setClassLoader(loader);
+ }
+ }
+
+ Deserializer s = SerDeUtils.getDeserializer(conf, tbl, false);
+ return SerDeUtils.getFieldsFromDeserializer(tbl.getTableName(), s);
+ } catch (Exception e) {
+ StringUtils.stringifyException(e);
+ throw new MetaException(e.getMessage());
+ } finally {
+ if (orgHiveLoader != null) {
+ conf.setClassLoader(orgHiveLoader);
+ }
+ }
+ }
+}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/StorageSchemaReader.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/StorageSchemaReader.java
new file mode 100644
index 0000000000000000000000000000000000000000..539604ca8bcdc1c88fdd37691c6dcb09284edeb0
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/reader/StorageSchemaReader.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.schema.reader;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Table;
+
+import java.util.List;
+
+/**
+ * An interface to implement reading schemas from stored data.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving public interface StorageSchemaReader {
+ /**
+ * Read the schema from the storage representation of the table.
+ * @param tbl metastore table object
+ * @param envContext environment context
+ * @param conf current configuration file
+ * @return list of field schemas
+ * @throws MetaException if the table storage could not be read
+ */
+ List readSchema(Table tbl, EnvironmentContext envContext, Configuration conf)
+ throws MetaException;
+}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/AvroFieldSchemaGenerator.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/AvroFieldSchemaGenerator.java
new file mode 100644
index 0000000000000000000000000000000000000000..f064e18c064b308d780abd711cabb5e7baa2e0a4
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/AvroFieldSchemaGenerator.java
@@ -0,0 +1,85 @@
+package org.apache.hadoop.hive.metastore.schema.utils;
+
+import org.apache.avro.Schema;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.serde2.avro.AvroSerdeException;
+import org.apache.hadoop.hive.serde2.avro.SchemaToTypeInfo;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class AvroFieldSchemaGenerator {
+ final private List columnNames;
+ final private List columnTypes;
+ final private List columnComments;
+
+ public AvroFieldSchemaGenerator(Schema schema) throws AvroSerdeException {
+ verifySchemaIsARecord(schema);
+
+ this.columnNames = generateColumnNames(schema);
+ this.columnTypes = SchemaToTypeInfo.generateColumnTypes(schema);
+ this.columnComments = generateColumnComments(schema);
+ assert columnNames.size() == columnTypes.size();
+ }
+
+ private static void verifySchemaIsARecord(Schema schema) throws AvroSerdeException {
+ if(!schema.getType().equals(Schema.Type.RECORD)) {
+ throw new AvroSerdeException("Schema for table must be of type RECORD. " +
+ "Received type: " + schema.getType());
+ }
+ }
+
+ private static List generateColumnNames(Schema schema) {
+ List fields = schema.getFields();
+ List fieldsList = new ArrayList(fields.size());
+
+ for (Schema.Field field : fields) {
+ fieldsList.add(field.name());
+ }
+
+ return fieldsList;
+ }
+
+ private static List generateColumnComments(Schema schema) {
+ List fields = schema.getFields();
+ List fieldComments = new ArrayList(fields.size());
+
+ for (Schema.Field field : fields) {
+ String fieldComment = field.doc() == null ? "" : field.doc();
+ fieldComments.add(fieldComment);
+ }
+
+ return fieldComments;
+ }
+
+ public List getFieldSchemas() throws AvroSerdeException {
+ int len = columnNames.size();
+ List fieldSchemas = new ArrayList<>(len);
+ for(int i = 0; i getFieldsFromAvroSchema(Configuration configuration,
+ Properties properties) throws AvroSerdeException, IOException {
+ // Reset member variables so we don't get in a half-constructed state
+ Schema schema = null;
+ List columnNames = null;
+ List columnTypes = null;
+
+ final String columnNameProperty = properties.getProperty(serdeConstants.LIST_COLUMNS);
+ final String columnTypeProperty = properties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
+ final String columnCommentProperty = properties.getProperty(LIST_COLUMN_COMMENTS,"");
+ final String columnNameDelimiter = properties.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? properties
+ .getProperty(serdeConstants.COLUMN_NAME_DELIMITER) : String.valueOf(SerDeUtils.COMMA);
+
+ if (hasExternalSchema(properties)
+ || columnNameProperty == null || columnNameProperty.isEmpty()
+ || columnTypeProperty == null || columnTypeProperty.isEmpty()) {
+ schema = AvroSchemaUtils.determineSchemaOrThrowException(configuration, properties);
+ } else {
+ // Get column names and sort order
+ columnNames = StringUtils.intern(
+ Arrays.asList(columnNameProperty.split(columnNameDelimiter)));
+ columnTypes = StorageSchemaUtils.getTypeInfosFromTypeString(columnTypeProperty);
+
+ schema = getSchemaFromCols(properties, columnNames, columnTypes, columnCommentProperty);
+ properties.setProperty(AvroTableProperties.SCHEMA_LITERAL.getPropName(), schema.toString());
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Avro schema is " + schema);
+ }
+
+ if (configuration == null) {
+ LOG.debug("Configuration null, not inserting schema");
+ } else {
+ configuration.set(
+ AvroTableProperties.AVRO_SERDE_SCHEMA.getPropName(), schema.toString(false));
+ }
+ return new AvroFieldSchemaGenerator(schema).getFieldSchemas();
+ }
+
+
+ private static boolean hasExternalSchema(Properties properties) {
+ return properties.getProperty(AvroTableProperties.SCHEMA_LITERAL.getPropName()) != null
+ || properties.getProperty(AvroTableProperties.SCHEMA_URL.getPropName()) != null;
+ }
+
+ private boolean supportedCategories(TypeInfo ti) {
+ final ObjectInspector.Category c = ti.getCategory();
+ return c.equals(ObjectInspector.Category.PRIMITIVE) ||
+ c.equals(ObjectInspector.Category.MAP) ||
+ c.equals(ObjectInspector.Category.LIST) ||
+ c.equals(ObjectInspector.Category.STRUCT) ||
+ c.equals(ObjectInspector.Category.UNION);
+ }
+
+ /**
+ * Attempt to determine the schema via the usual means, but do not throw
+ * an exception if we fail. Instead, signal failure via a special
+ * schema.
+ */
+ public static Schema determineSchemaOrReturnErrorSchema(Configuration conf, Properties props) {
+ try {
+ return AvroSchemaUtils.determineSchemaOrThrowException(conf, props);
+ } catch(AvroSerdeException he) {
+ LOG.warn("Encountered AvroSerdeException determining schema. Returning " +
+ "signal schema to indicate problem", he);
+ } catch (Exception e) {
+ LOG.warn("Encountered exception determining schema. Returning signal " +
+ "schema to indicate problem", e);
+ }
+ return SchemaResolutionProblem.SIGNAL_BAD_SCHEMA;
+ }
+
+ /**
+ * Determine the schema to that's been provided for Avro serde work.
+ * @param properties containing a key pointing to the schema, one way or another
+ * @return schema to use while serdeing the avro file
+ * @throws IOException if error while trying to read the schema from another location
+ * @throws AvroSerdeException if unable to find a schema or pointer to it in the properties
+ */
+ public static Schema determineSchemaOrThrowException(Configuration conf, Properties properties)
+ throws IOException, AvroSerdeException {
+ String schemaString = properties.getProperty(AvroTableProperties.SCHEMA_LITERAL.getPropName());
+ if(schemaString != null && !schemaString.equals(SCHEMA_NONE))
+ return AvroSchemaUtils.getSchemaFor(schemaString);
+
+ // Try pulling directly from URL
+ schemaString = properties.getProperty(AvroTableProperties.SCHEMA_URL.getPropName());
+ if (schemaString == null) {
+ final String columnNameProperty = properties.getProperty(serdeConstants.LIST_COLUMNS);
+ final String columnTypeProperty = properties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
+ final String columnCommentProperty = properties.getProperty(LIST_COLUMN_COMMENTS);
+ if (columnNameProperty == null || columnNameProperty.isEmpty()
+ || columnTypeProperty == null || columnTypeProperty.isEmpty() ) {
+ throw new AvroSerdeException(EXCEPTION_MESSAGE);
+ }
+ final String columnNameDelimiter = properties.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? properties
+ .getProperty(serdeConstants.COLUMN_NAME_DELIMITER) : String.valueOf(SerDeUtils.COMMA);
+ // Get column names and types
+ List columnNames = Arrays.asList(columnNameProperty.split(columnNameDelimiter));
+ List columnTypes = StorageSchemaUtils.getTypeInfosFromTypeString(columnTypeProperty);
+
+ Schema schema = getSchemaFromCols(properties, columnNames, columnTypes, columnCommentProperty);
+ properties.setProperty(AvroTableProperties.SCHEMA_LITERAL.getPropName(), schema.toString());
+ if (conf != null)
+ conf.set(AvroTableProperties.AVRO_SERDE_SCHEMA.getPropName(), schema.toString(false));
+ return schema;
+ } else if(schemaString.equals(SCHEMA_NONE)) {
+ throw new AvroSerdeException(EXCEPTION_MESSAGE);
+ }
+
+ try {
+ Schema s = getSchemaFromFS(schemaString, conf);
+ if (s == null) {
+ //in case schema is not a file system
+ return AvroSchemaUtils.getSchemaFor(new URL(schemaString));
+ }
+ return s;
+ } catch (IOException ioe) {
+ throw new AvroSerdeException("Unable to read schema from given path: " + schemaString, ioe);
+ } catch (URISyntaxException urie) {
+ throw new AvroSerdeException("Unable to read schema from given path: " + schemaString, urie);
+ }
+ }
+
+ // Protected for testing and so we can pass in a conf for testing.
+ protected static Schema getSchemaFromFS(String schemaFSUrl,
+ Configuration conf) throws IOException, URISyntaxException {
+ FSDataInputStream in = null;
+ FileSystem fs = null;
+ try {
+ fs = FileSystem.get(new URI(schemaFSUrl), conf);
+ } catch (IOException ioe) {
+ //return null only if the file system in schema is not recognized
+ if (LOG.isDebugEnabled()) {
+ String msg = "Failed to open file system for uri " + schemaFSUrl + " assuming it is not a FileSystem url";
+ LOG.debug(msg, ioe);
+ }
+
+ return null;
+ }
+ try {
+ in = fs.open(new Path(schemaFSUrl));
+ Schema s = AvroSchemaUtils.getSchemaFor(in);
+ return s;
+ } finally {
+ if(in != null) in.close();
+ }
+ }
+
+ public static Schema getSchemaFor(File file) {
+ Schema.Parser parser = new Schema.Parser();
+ Schema schema;
+ try {
+ schema = parser.parse(file);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to parse Avro schema from " + file.getName(), e);
+ }
+ return schema;
+ }
+
+ public static Schema getSchemaFor(InputStream stream) {
+ Schema.Parser parser = new Schema.Parser();
+ Schema schema;
+ try {
+ schema = parser.parse(stream);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to parse Avro schema", e);
+ }
+ return schema;
+ }
+
+ public static Schema getSchemaFor(String str) {
+ Schema.Parser parser = new Schema.Parser();
+ Schema schema = parser.parse(str);
+ return schema;
+ }
+
+ public static Schema getSchemaFor(URL url) {
+ InputStream in = null;
+ try {
+ in = url.openStream();
+ return getSchemaFor(in);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to parse Avro schema", e);
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ // Ignore
+ }
+ }
+ }
+ }
+
+ public static Schema getSchemaFromCols(Properties properties,
+ List columnNames, List columnTypes, String columnCommentProperty) {
+ List columnComments;
+ if (columnCommentProperty == null || columnCommentProperty.isEmpty()) {
+ columnComments = new ArrayList();
+ } else {
+ //Comments are separated by "\0" in columnCommentProperty, see method getSchema
+ //in MetaStoreUtils where this string columns.comments is generated
+ columnComments = Arrays.asList(columnCommentProperty.split("\0"));
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("columnComments is " + columnCommentProperty);
+ }
+ }
+ if (columnNames.size() != columnTypes.size()) {
+ throw new IllegalArgumentException("getSchemaFromCols initialization failed. Number of column " +
+ "name and column type differs. columnNames = " + columnNames + ", columnTypes = " +
+ columnTypes);
+ }
+
+ final String tableName = properties.getProperty(AvroSerDeConstants.TABLE_NAME);
+ final String tableComment = properties.getProperty(AvroSerDeConstants.TABLE_COMMENT);
+ TypeInfoToSchema typeInfoToSchema = new TypeInfoToSchema();
+ return typeInfoToSchema.convert(columnNames, columnTypes, columnComments,
+ properties.getProperty(AvroTableProperties.SCHEMA_NAMESPACE.getPropName()),
+ properties.getProperty(AvroTableProperties.SCHEMA_NAME.getPropName(), tableName),
+ properties.getProperty(AvroTableProperties.SCHEMA_DOC.getPropName(), tableComment));
+
+ }
+
+ /**
+ * Determine if an Avro schema is of type Union[T, NULL]. Avro supports nullable
+ * types via a union of type T and null. This is a very common use case.
+ * As such, we want to silently convert it to just T and allow the value to be null.
+ *
+ * When a Hive union type is used with AVRO, the schema type becomes
+ * Union[NULL, T1, T2, ...]. The NULL in the union should be silently removed
+ *
+ * @return true if type represents Union[T, Null], false otherwise
+ */
+ public static boolean isNullableType(Schema schema) {
+ if (!schema.getType().equals(Schema.Type.UNION)) {
+ return false;
+ }
+
+ List itemSchemas = schema.getTypes();
+ if (itemSchemas.size() < 2) {
+ return false;
+ }
+
+ for (Schema itemSchema : itemSchemas) {
+ if (Schema.Type.NULL.equals(itemSchema.getType())) {
+ return true;
+ }
+ }
+
+ // [null, null] not allowed, so this check is ok.
+ return false;
+ }
+
+ /**
+ * In a nullable type, get the schema for the non-nullable type. This method
+ * does no checking that the provides Schema is nullable.
+ */
+ public static Schema getOtherTypeFromNullableType(Schema schema) {
+ List itemSchemas = new ArrayList<>();
+ for (Schema itemSchema : schema.getTypes()) {
+ if (!Schema.Type.NULL.equals(itemSchema.getType())) {
+ itemSchemas.add(itemSchema);
+ }
+ }
+
+ if (itemSchemas.size() > 1) {
+ return Schema.createUnion(itemSchemas);
+ } else {
+ return itemSchemas.get(0);
+ }
+ }
+}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/SchemaResolutionProblem.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/SchemaResolutionProblem.java
new file mode 100644
index 0000000000000000000000000000000000000000..3f93c01cc375ec40deb88477252ee2e3eb061b81
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/SchemaResolutionProblem.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.schema.utils;
+
+import org.apache.avro.Schema;
+
+/**
+ * This class is copied from SchemaResolutionProblem from the Hive source code as part of metastore
+ * separation
+ */
+class SchemaResolutionProblem {
+ static final String sentinelString = "{\n" +
+ " \"namespace\": \"org.apache.hadoop.hive\",\n" +
+ " \"name\": \"CannotDetermineSchemaSentinel\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"ERROR_ERROR_ERROR_ERROR_ERROR_ERROR_ERROR\",\n" +
+ " \"type\":\"string\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"Cannot_determine_schema\",\n" +
+ " \"type\":\"string\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"check\",\n" +
+ " \"type\":\"string\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"schema\",\n" +
+ " \"type\":\"string\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"url\",\n" +
+ " \"type\":\"string\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"and\",\n" +
+ " \"type\":\"string\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"literal\",\n" +
+ " \"type\":\"string\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ public final static Schema SIGNAL_BAD_SCHEMA = AvroSchemaUtils.getSchemaFor(sentinelString);
+}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/SerDeUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/SerDeUtils.java
new file mode 100644
index 0000000000000000000000000000000000000000..5773ccf0b2387533af1b9f1e33545b03c514591c
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/SerDeUtils.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.schema.utils;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.utils.JavaUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
+import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+
+public class SerDeUtils {
+
+ public static final char COMMA = ',';
+ private static final String FROM_SERIALIZER = "from deserializer";
+ private static final Logger LOG = LoggerFactory.getLogger(SerDeUtils.class.getName());
+
+ private static String determineFieldComment(String comment) {
+ return (comment == null) ? FROM_SERIALIZER : comment;
+ }
+
+ /**
+ * Initializes a SerDe.
+ * @param deserializer
+ * @param conf
+ * @param tblProps
+ * @param partProps
+ * @throws SerDeException
+ */
+ public static void initializeSerDeWithoutErrorCheck(Deserializer deserializer,
+ Configuration conf, Properties tblProps,
+ Properties partProps) throws SerDeException {
+ if (deserializer instanceof AbstractSerDe) {
+ ((AbstractSerDe) deserializer).initialize(conf, tblProps, partProps);
+ } else {
+ deserializer.initialize(conf, createOverlayedProperties(tblProps, partProps));
+ }
+ }
+
+ /**
+ * Returns the union of table and partition properties,
+ * with partition properties taking precedence.
+ * @param tblProps
+ * @param partProps
+ * @return the overlayed properties
+ */
+ public static Properties createOverlayedProperties(Properties tblProps, Properties partProps) {
+ Properties props = new Properties();
+ props.putAll(tblProps);
+ if (partProps != null) {
+ props.putAll(partProps);
+ }
+ return props;
+ }
+
+ /**
+ * @param tableName name of the table
+ * @param deserializer deserializer to use
+ * @return the list of fields
+ * @throws SerDeException if the serde throws an exception
+ * @throws MetaException if one of the fields or types in the table is invalid
+ */
+ public static List getFieldsFromDeserializer(String tableName,
+ Deserializer deserializer) throws SerDeException, MetaException {
+ ObjectInspector oi = deserializer.getObjectInspector();
+ String[] names = tableName.split("\\.");
+ String last_name = names[names.length - 1];
+ for (int i = 1; i < names.length; i++) {
+
+ if (oi instanceof StructObjectInspector) {
+ StructObjectInspector soi = (StructObjectInspector) oi;
+ StructField sf = soi.getStructFieldRef(names[i]);
+ if (sf == null) {
+ throw new MetaException("Invalid Field " + names[i]);
+ } else {
+ oi = sf.getFieldObjectInspector();
+ }
+ } else if (oi instanceof ListObjectInspector
+ && names[i].equalsIgnoreCase("$elem$")) {
+ ListObjectInspector loi = (ListObjectInspector) oi;
+ oi = loi.getListElementObjectInspector();
+ } else if (oi instanceof MapObjectInspector
+ && names[i].equalsIgnoreCase("$key$")) {
+ MapObjectInspector moi = (MapObjectInspector) oi;
+ oi = moi.getMapKeyObjectInspector();
+ } else if (oi instanceof MapObjectInspector
+ && names[i].equalsIgnoreCase("$value$")) {
+ MapObjectInspector moi = (MapObjectInspector) oi;
+ oi = moi.getMapValueObjectInspector();
+ } else {
+ throw new MetaException("Unknown type for " + names[i]);
+ }
+ }
+
+ ArrayList str_fields = new ArrayList<>();
+ // rules on how to recurse the ObjectInspector based on its type
+ if (oi.getCategory() != Category.STRUCT) {
+ str_fields.add(new FieldSchema(last_name, oi.getTypeName(),
+ FROM_SERIALIZER));
+ } else {
+ List extends StructField> fields = ((StructObjectInspector) oi)
+ .getAllStructFieldRefs();
+ for (int i = 0; i < fields.size(); i++) {
+ StructField structField = fields.get(i);
+ String fieldName = structField.getFieldName();
+ String fieldTypeName = structField.getFieldObjectInspector().getTypeName();
+ String fieldComment = determineFieldComment(structField.getFieldComment());
+
+ str_fields.add(new FieldSchema(fieldName, fieldTypeName, fieldComment));
+ }
+ }
+ return str_fields;
+ }
+
+ /**
+ * Initializes a SerDe.
+ * @param deserializer
+ * @param conf
+ * @param tblProps
+ * @param partProps
+ * @throws SerDeException
+ */
+ public static void initializeSerDe(Deserializer deserializer, Configuration conf,
+ Properties tblProps, Properties partProps)
+ throws SerDeException {
+ if (deserializer instanceof AbstractSerDe) {
+ ((AbstractSerDe) deserializer).initialize(conf, tblProps, partProps);
+ String msg = ((AbstractSerDe) deserializer).getConfigurationErrors();
+ if (msg != null && !msg.isEmpty()) {
+ throw new SerDeException(msg);
+ }
+ } else {
+ deserializer.initialize(conf, createOverlayedProperties(tblProps, partProps));
+ }
+ }
+
+
+ /**
+ * getDeserializer
+ *
+ * Get the Deserializer for a table.
+ *
+ * @param conf
+ * - hadoop config
+ * @param table
+ * the table
+ * @return
+ * Returns instantiated deserializer by looking up class name of deserializer stored in
+ * storage descriptor of passed in table. Also, initializes the deserializer with schema
+ * of table.
+ * @exception MetaException
+ * if any problems instantiating the Deserializer
+ *
+ * todo - this should move somewhere into serde.jar
+ *
+ */
+ static public Deserializer getDeserializer(Configuration conf,
+ org.apache.hadoop.hive.metastore.api.Table table, boolean skipConfError) throws
+ MetaException {
+ String lib = table.getSd().getSerdeInfo().getSerializationLib();
+ if (lib == null) {
+ return null;
+ }
+ return getDeserializer(conf, table, skipConfError, lib);
+ }
+
+ public static Deserializer getDeserializer(Configuration conf,
+ org.apache.hadoop.hive.metastore.api.Table table, boolean skipConfError,
+ String lib) throws MetaException {
+ try {
+ Deserializer deserializer = JavaUtils.newInstance(conf.getClassByName(lib).
+ asSubclass(Deserializer.class), conf);
+ if (skipConfError) {
+ SerDeUtils.initializeSerDeWithoutErrorCheck(deserializer, conf,
+ MetaStoreUtils.getTableMetadata(table), null);
+ } else {
+ SerDeUtils.initializeSerDe(deserializer, conf, MetaStoreUtils.getTableMetadata(table), null);
+ }
+ return deserializer;
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Throwable e) {
+ LOG.error("error in initSerDe: " + e.getClass().getName() + " "
+ + e.getMessage(), e);
+ throw new MetaException(e.getClass().getName() + " " + e.getMessage());
+ }
+ }
+}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/StorageSchemaUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/StorageSchemaUtils.java
new file mode 100644
index 0000000000000000000000000000000000000000..1fc4e1ad5be31e00497f9487324b34c5b013374d
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/schema/utils/StorageSchemaUtils.java
@@ -0,0 +1,141 @@
+package org.apache.hadoop.hive.metastore.schema.utils;
+
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveTypeEntry;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoParser;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.ByteWritable;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.ShortWritable;
+import org.apache.hadoop.io.Text;
+
+import java.sql.Date;
+import java.sql.Timestamp;
+import java.util.List;
+
+public class StorageSchemaUtils {
+
+ public static final PrimitiveTypeEntry binaryTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.BINARY,
+ serdeConstants.BINARY_TYPE_NAME, byte[].class, byte[].class, BytesWritable.class);
+ public static final PrimitiveTypeEntry stringTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.STRING,
+ serdeConstants.STRING_TYPE_NAME, null, String.class, Text.class);
+ public static final PrimitiveTypeEntry booleanTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN,
+ serdeConstants.BOOLEAN_TYPE_NAME, Boolean.TYPE, Boolean.class, BooleanWritable.class);
+ public static final PrimitiveTypeEntry intTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.INT,
+ serdeConstants.INT_TYPE_NAME, Integer.TYPE, Integer.class, IntWritable.class);
+ public static final PrimitiveTypeEntry longTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.LONG,
+ serdeConstants.BIGINT_TYPE_NAME, Long.TYPE, Long.class, LongWritable.class);
+ public static final PrimitiveTypeEntry floatTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.FLOAT,
+ serdeConstants.FLOAT_TYPE_NAME, Float.TYPE, Float.class, FloatWritable.class);
+ public static final PrimitiveTypeEntry voidTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.VOID,
+ serdeConstants.VOID_TYPE_NAME, Void.TYPE, Void.class, NullWritable.class);
+
+ public static final PrimitiveTypeEntry doubleTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.DOUBLE,
+ serdeConstants.DOUBLE_TYPE_NAME, Double.TYPE, Double.class, DoubleWritable.class);
+ public static final PrimitiveTypeEntry byteTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.BYTE,
+ serdeConstants.TINYINT_TYPE_NAME, Byte.TYPE, Byte.class, ByteWritable.class);
+ public static final PrimitiveTypeEntry shortTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.SHORT,
+ serdeConstants.SMALLINT_TYPE_NAME, Short.TYPE, Short.class, ShortWritable.class);
+ public static final PrimitiveTypeEntry dateTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.DATE,
+ serdeConstants.DATE_TYPE_NAME, null, Date.class, DateWritable.class);
+
+ public static final PrimitiveTypeEntry timestampTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP,
+ //serdeConstants.TIMESTAMP_TYPE_NAME, null, Timestamp.class, TimestampWritable.class);
+ //TODO need to move TimestampWritable to storage-api to make this work
+ serdeConstants.TIMESTAMP_TYPE_NAME, null, Timestamp.class, null);
+ public static final PrimitiveTypeEntry timestampTZTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMPLOCALTZ,
+ //serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME, null, TimestampTZ.class,
+ //TimestampLocalTZWritable.class);
+ //TODO need to move TimestampTZ and TimestampLocalTZWritable to storage-api to make this work
+ serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME, null, null,
+ null);
+ public static final PrimitiveTypeEntry intervalYearMonthTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.INTERVAL_YEAR_MONTH,
+ //serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME, null, HiveIntervalYearMonth.class,
+ //HiveIntervalYearMonthWritable.class);
+ //TODO need to move HiveIntervalYearMonth and HiveIntervalYearMonthWritable to storage-api to make this work
+ serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME, null, null,
+ null);
+ public static final PrimitiveTypeEntry intervalDayTimeTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.INTERVAL_DAY_TIME,
+ serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME, null, HiveIntervalDayTime.class,
+ //HiveIntervalDayTimeWritable.class);
+ //TODO need to move HiveIntervalYearMonth and HiveIntervalDayTimeWritable to storage-api to make this work
+ null);
+ public static final PrimitiveTypeEntry decimalTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.DECIMAL,
+ serdeConstants.DECIMAL_TYPE_NAME, null, HiveDecimal.class, HiveDecimalWritable.class);
+ public static final PrimitiveTypeEntry varcharTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.VARCHAR,
+ serdeConstants.VARCHAR_TYPE_NAME, null, HiveVarchar.class, HiveVarcharWritable.class);
+ public static final PrimitiveTypeEntry charTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.CHAR,
+ serdeConstants.CHAR_TYPE_NAME, null, HiveChar.class, HiveCharWritable.class);
+
+ // The following is a complex type for special handling
+ public static final PrimitiveTypeEntry unknownTypeEntry =
+ new PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory.UNKNOWN, "unknown", null,
+ Object.class, null);
+
+
+ static {
+ PrimitiveTypeEntry.registerType(binaryTypeEntry);
+ PrimitiveTypeEntry.registerType(stringTypeEntry);
+ PrimitiveTypeEntry.registerType(charTypeEntry);
+ PrimitiveTypeEntry.registerType(varcharTypeEntry);
+ PrimitiveTypeEntry.registerType(booleanTypeEntry);
+ PrimitiveTypeEntry.registerType(intTypeEntry);
+ PrimitiveTypeEntry.registerType(longTypeEntry);
+ PrimitiveTypeEntry.registerType(floatTypeEntry);
+ PrimitiveTypeEntry.registerType(voidTypeEntry);
+ PrimitiveTypeEntry.registerType(doubleTypeEntry);
+ PrimitiveTypeEntry.registerType(byteTypeEntry);
+ PrimitiveTypeEntry.registerType(shortTypeEntry);
+ PrimitiveTypeEntry.registerType(dateTypeEntry);
+ PrimitiveTypeEntry.registerType(timestampTypeEntry);
+ PrimitiveTypeEntry.registerType(timestampTZTypeEntry);
+ PrimitiveTypeEntry.registerType(intervalYearMonthTypeEntry);
+ PrimitiveTypeEntry.registerType(intervalDayTimeTypeEntry);
+ PrimitiveTypeEntry.registerType(decimalTypeEntry);
+ PrimitiveTypeEntry.registerType(unknownTypeEntry);
+ }
+
+
+ public static final char COMMA = ',';
+ public static List getTypeInfosFromTypeString(String columnTypeProperty) {
+ return new TypeInfoParser(columnTypeProperty).parseTypeInfos();
+ }
+
+ private static final String FROM_STORAGE_SCHEMA_READER = "generated by storage schema reader";
+ public static String determineFieldComment(String comment) {
+ return (comment == null) ? FROM_STORAGE_SCHEMA_READER : comment;
+ }
+}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java
index b08d9fd71f6c27abcda5561adeef755d6f3fc003..60682e24afc85b73a49bb83b8650dea43b669016 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java
@@ -17,16 +17,38 @@
*/
package org.apache.hadoop.hive.metastore.utils;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.UnknownHostException;
public class JavaUtils {
public static final Logger LOG = LoggerFactory.getLogger(JavaUtils.class);
+ private static final Method configureMethod;
+ private static final Class> jobConfClass, jobConfigurableClass;
+
+ static {
+ Class> jobConfClassLocal, jobConfigurableClassLocal;
+ Method configureMethodLocal;
+ try {
+ jobConfClassLocal = Class.forName("org.apache.hadoop.mapred.JobConf");
+ jobConfigurableClassLocal = Class.forName("org.apache.hadoop.mapred.JobConfigurable");
+ configureMethodLocal = jobConfigurableClassLocal.getMethod("configure", jobConfClassLocal);
+ } catch (Throwable t) {
+ // Meh.
+ jobConfClassLocal = jobConfigurableClassLocal = null;
+ configureMethodLocal = null;
+ }
+ jobConfClass = jobConfClassLocal;
+ jobConfigurableClass = jobConfigurableClassLocal;
+ configureMethod = configureMethodLocal;
+ }
/**
* Standard way of getting classloader in Hive code (outside of Hadoop).
@@ -100,8 +122,55 @@ public static ClassLoader getClassLoader() {
throw new RuntimeException("Unable to instantiate " + theClass.getName(), e);
}
}
+ private static final Class>[] EMPTY_ARRAY = new Class[] {};
+ /**
+ * Create an object for the given class and initialize it from conf
+ * @param theClass class of which an object is created
+ * @param conf Configuration
+ * @return a new object
+ */
+ @SuppressWarnings("unchecked")
+ public static T newInstance(Class theClass, Configuration conf) {
+ T result;
+ try {
+ // TODO Do we need a constructor cache like Hive here?
+ Constructor> ctor = theClass.getDeclaredConstructor(EMPTY_ARRAY);
+ ctor.setAccessible(true);
+ result = (T)ctor.newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ setConf(result, conf);
+ return result;
+ }
/**
+ * Check and set 'configuration' if necessary.
+ *
+ * @param theObject object for which to set configuration
+ * @param conf Configuration
+ */
+ public static void setConf(Object theObject, Configuration conf) {
+ if (conf != null) {
+ if (theObject instanceof Configurable) {
+ ((Configurable) theObject).setConf(conf);
+ }
+ setJobConf(theObject, conf);
+ }
+ }
+
+ private static void setJobConf(Object theObject, Configuration conf) {
+ if (configureMethod == null) return;
+ try {
+ if (jobConfClass.isAssignableFrom(conf.getClass()) &&
+ jobConfigurableClass.isAssignableFrom(theObject.getClass())) {
+ configureMethod.invoke(theObject, conf);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Error in configuring object", e);
+ }
+ }
+ /**
* @return name of current host
*/
public static String hostname() {
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
index cde34bcf42bdb49f7e4d11b9274337ad1b4c3285..8e52779d8fc6f53dc0fe64467c4f40b1d1dc3eb9 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
@@ -55,6 +55,7 @@
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
+import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.security.SaslRpcServer;
import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
@@ -1560,4 +1561,5 @@ public static boolean isMaterializedViewTable(Table table) {
}
return cols;
}
+
}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerDeConstants.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerDeConstants.java
new file mode 100644
index 0000000000000000000000000000000000000000..e875c13a1af3733fcb76f941144519a77c46c8c6
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerDeConstants.java
@@ -0,0 +1,25 @@
+package org.apache.hadoop.hive.serde2.avro;
+
+/**
+ * This class contains some of the constants which are specific to AvroSerDe
+ * They should always match with the constants defined in AvroSerDe.java in Hive Source code. These
+ * constants were copied as part of separating metastore from Hive.
+ */
+public class AvroSerDeConstants {
+ public static final String TABLE_NAME = "name";
+ public static final String TABLE_COMMENT = "comment";
+ public static final String LIST_COLUMN_COMMENTS = "columns.comments";
+
+ public static final String DECIMAL_TYPE_NAME = "decimal";
+ public static final String CHAR_TYPE_NAME = "char";
+ public static final String VARCHAR_TYPE_NAME = "varchar";
+ public static final String DATE_TYPE_NAME = "date";
+ public static final String TIMESTAMP_TYPE_NAME = "timestamp-millis";
+ public static final String AVRO_PROP_LOGICAL_TYPE = "logicalType";
+ public static final String AVRO_PROP_PRECISION = "precision";
+ public static final String AVRO_PROP_SCALE = "scale";
+ public static final String AVRO_PROP_MAX_LENGTH = "maxLength";
+ public static final String AVRO_STRING_TYPE_NAME = "string";
+ public static final String AVRO_INT_TYPE_NAME = "int";
+ public static final String AVRO_LONG_TYPE_NAME = "long";
+}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeException.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeException.java
new file mode 100644
index 0000000000000000000000000000000000000000..bbf4cc1078f2a9d1395723c395228db865ca850d
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeException.java
@@ -0,0 +1,11 @@
+package org.apache.hadoop.hive.serde2.avro;
+
+public class AvroSerdeException extends Exception {
+ public AvroSerdeException(String s, Exception ex) {
+ super(s, ex);
+ }
+
+ public AvroSerdeException(String msg) {
+ super(msg);
+ }
+}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..acc693c7b7cfecff08ce70fc918ed2276c0518f0
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.avro;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Cache for objects whose creation only depends on some other set of objects and therefore can be
+ * used against other equivalent versions of those objects. Essentially memoizes instance creation.
+ *
+ * @param Object that determines the instance. The cache uses this object as a key for
+ * its hash which is why it is imperative to have appropriate equals and hashcode
+ * implementation for this object for the cache to work properly
+ * @param Instance that will be created from SeedObject.
+ */
+public abstract class InstanceCache {
+ private static final Logger LOG = LoggerFactory.getLogger(InstanceCache.class);
+ Map cache = new HashMap();
+
+ public InstanceCache() {}
+
+ /**
+ * Retrieve (or create if it doesn't exist) the correct Instance for this
+ * SeedObject
+ */
+ public Instance retrieve(SeedObject hv) throws AvroSerdeException {
+ return retrieve(hv, null);
+ }
+
+ /**
+ * Retrieve (or create if it doesn't exist) the correct Instance for this
+ * SeedObject using 'seenSchemas' to resolve circular references
+ */
+ public synchronized Instance retrieve(SeedObject hv,
+ Set seenSchemas) throws AvroSerdeException {
+ if(LOG.isDebugEnabled()) LOG.debug("Checking for hv: " + hv.toString());
+
+ if(cache.containsKey(hv)) {
+ if(LOG.isDebugEnabled()) LOG.debug("Returning cache result.");
+ return cache.get(hv);
+ }
+
+ if(LOG.isDebugEnabled()) LOG.debug("Creating new instance and storing in cache");
+
+ Instance instance = makeInstance(hv, seenSchemas);
+ cache.put(hv, instance);
+ return instance;
+ }
+
+ protected abstract Instance makeInstance(SeedObject hv,
+ Set seenSchemas) throws AvroSerdeException;
+}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java
new file mode 100644
index 0000000000000000000000000000000000000000..29298b0e2c2964634b029c590357e6c91d102635
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.avro;
+
+import static org.apache.avro.Schema.Type.BOOLEAN;
+import static org.apache.avro.Schema.Type.BYTES;
+import static org.apache.avro.Schema.Type.DOUBLE;
+import static org.apache.avro.Schema.Type.FIXED;
+import static org.apache.avro.Schema.Type.FLOAT;
+import static org.apache.avro.Schema.Type.INT;
+import static org.apache.avro.Schema.Type.LONG;
+import static org.apache.avro.Schema.Type.NULL;
+import static org.apache.avro.Schema.Type.STRING;
+
+import org.apache.avro.Schema;
+import org.apache.hadoop.hive.metastore.schema.utils.AvroSchemaUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfoValidationUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Hashtable;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Convert an Avro Schema to a Hive TypeInfo. This class is copied from Hive source code for
+ * standalone metastore
+ */
+public class SchemaToTypeInfo {
+ // Conversion of Avro primitive types to Hive primitive types
+ // Avro Hive
+ // Null
+ // boolean boolean check
+ // int int check
+ // long bigint check
+ // float double check
+ // double double check
+ // bytes binary check
+ // fixed binary check
+ // string string check
+ // tinyint
+ // smallint
+
+ // Map of Avro's primitive types to Hives (for those that are supported by both)
+ private static final Map primitiveTypeToTypeInfo = initTypeMap();
+ private static Map initTypeMap() {
+ Map theMap = new Hashtable();
+ theMap.put(NULL, TypeInfoFactory.getPrimitiveTypeInfo("void"));
+ theMap.put(BOOLEAN, TypeInfoFactory.getPrimitiveTypeInfo("boolean"));
+ theMap.put(INT, TypeInfoFactory.getPrimitiveTypeInfo("int"));
+ theMap.put(LONG, TypeInfoFactory.getPrimitiveTypeInfo("bigint"));
+ theMap.put(FLOAT, TypeInfoFactory.getPrimitiveTypeInfo("float"));
+ theMap.put(DOUBLE, TypeInfoFactory.getPrimitiveTypeInfo("double"));
+ theMap.put(BYTES, TypeInfoFactory.getPrimitiveTypeInfo("binary"));
+ theMap.put(FIXED, TypeInfoFactory.getPrimitiveTypeInfo("binary"));
+ theMap.put(STRING, TypeInfoFactory.getPrimitiveTypeInfo("string"));
+ return Collections.unmodifiableMap(theMap);
+ }
+
+ /**
+ * Generate a list of of TypeInfos from an Avro schema. This method is
+ * currently public due to some weirdness in deserializing unions, but
+ * will be made private once that is resolved.
+ * @param schema Schema to generate field types for
+ * @return List of TypeInfos, each element of which is a TypeInfo derived
+ * from the schema.
+ * @throws AvroSerdeException for problems during conversion.
+ */
+ public static List generateColumnTypes(Schema schema) throws AvroSerdeException {
+ return generateColumnTypes (schema, null);
+ }
+
+ /**
+ * Generate a list of of TypeInfos from an Avro schema. This method is
+ * currently public due to some weirdness in deserializing unions, but
+ * will be made private once that is resolved.
+ * @param schema Schema to generate field types for
+ * @param seenSchemas stores schemas processed in the parsing done so far,
+ * helping to resolve circular references in the schema
+ * @return List of TypeInfos, each element of which is a TypeInfo derived
+ * from the schema.
+ * @throws AvroSerdeException for problems during conversion.
+ */
+ public static List generateColumnTypes(Schema schema,
+ Set seenSchemas) throws AvroSerdeException {
+ List fields = schema.getFields();
+
+ List types = new ArrayList(fields.size());
+
+ for (Schema.Field field : fields) {
+ types.add(generateTypeInfo(field.schema(), seenSchemas));
+ }
+
+ return types;
+ }
+
+ static InstanceCache typeInfoCache = new InstanceCache() {
+ @Override
+ protected TypeInfo makeInstance(Schema s,
+ Set seenSchemas)
+ throws AvroSerdeException {
+ return generateTypeInfoWorker(s, seenSchemas);
+ }
+ };
+ /**
+ * Convert an Avro Schema into an equivalent Hive TypeInfo.
+ * @param schema to record. Must be of record type.
+ * @param seenSchemas stores schemas processed in the parsing done so far,
+ * helping to resolve circular references in the schema
+ * @return TypeInfo matching the Avro schema
+ * @throws AvroSerdeException for any problems during conversion.
+ */
+ public static TypeInfo generateTypeInfo(Schema schema,
+ Set seenSchemas) throws AvroSerdeException {
+ // For bytes type, it can be mapped to decimal.
+ Schema.Type type = schema.getType();
+ if (type == BYTES && AvroSerDeConstants.DECIMAL_TYPE_NAME
+ .equalsIgnoreCase(schema.getProp(AvroSerDeConstants.AVRO_PROP_LOGICAL_TYPE))) {
+ int precision = 0;
+ int scale = 0;
+ try {
+ precision = schema.getJsonProp(AvroSerDeConstants.AVRO_PROP_PRECISION).getIntValue();
+ scale = schema.getJsonProp(AvroSerDeConstants.AVRO_PROP_SCALE).getIntValue();
+ } catch (Exception ex) {
+ throw new AvroSerdeException("Failed to obtain scale value from file schema: " + schema, ex);
+ }
+
+ try {
+ PrimitiveTypeInfoValidationUtils.validateParameter(precision, scale);
+ } catch (Exception ex) {
+ throw new AvroSerdeException("Invalid precision or scale for decimal type", ex);
+ }
+
+ return TypeInfoFactory.getDecimalTypeInfo(precision, scale);
+ }
+
+ if (type == STRING &&
+ AvroSerDeConstants.CHAR_TYPE_NAME.equalsIgnoreCase(schema.getProp(AvroSerDeConstants.AVRO_PROP_LOGICAL_TYPE))) {
+ int maxLength = 0;
+ try {
+ maxLength = schema.getJsonProp(AvroSerDeConstants.AVRO_PROP_MAX_LENGTH).getValueAsInt();
+ } catch (Exception ex) {
+ throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex);
+ }
+ return TypeInfoFactory.getCharTypeInfo(maxLength);
+ }
+
+ if (type == STRING && AvroSerDeConstants.VARCHAR_TYPE_NAME
+ .equalsIgnoreCase(schema.getProp(AvroSerDeConstants.AVRO_PROP_LOGICAL_TYPE))) {
+ int maxLength = 0;
+ try {
+ maxLength = schema.getJsonProp(AvroSerDeConstants.AVRO_PROP_MAX_LENGTH).getValueAsInt();
+ } catch (Exception ex) {
+ throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex);
+ }
+ return TypeInfoFactory.getVarcharTypeInfo(maxLength);
+ }
+
+ if (type == INT &&
+ AvroSerDeConstants.DATE_TYPE_NAME.equals(schema.getProp(AvroSerDeConstants.AVRO_PROP_LOGICAL_TYPE))) {
+ return TypeInfoFactory.dateTypeInfo;
+ }
+
+ if (type == LONG &&
+ AvroSerDeConstants.TIMESTAMP_TYPE_NAME.equals(schema.getProp(AvroSerDeConstants.AVRO_PROP_LOGICAL_TYPE))) {
+ return TypeInfoFactory.timestampTypeInfo;
+ }
+
+ return typeInfoCache.retrieve(schema, seenSchemas);
+ }
+
+ private static TypeInfo generateTypeInfoWorker(Schema schema,
+ Set seenSchemas) throws AvroSerdeException {
+ // Avro requires NULLable types to be defined as unions of some type T
+ // and NULL. This is annoying and we're going to hide it from the user.
+ if(AvroSchemaUtils.isNullableType(schema)) {
+ return generateTypeInfo(
+ AvroSchemaUtils.getOtherTypeFromNullableType(schema), seenSchemas);
+ }
+
+ Schema.Type type = schema.getType();
+ if(primitiveTypeToTypeInfo.containsKey(type)) {
+ return primitiveTypeToTypeInfo.get(type);
+ }
+
+ switch(type) {
+ case RECORD: return generateRecordTypeInfo(schema, seenSchemas);
+ case MAP: return generateMapTypeInfo(schema, seenSchemas);
+ case ARRAY: return generateArrayTypeInfo(schema, seenSchemas);
+ case UNION: return generateUnionTypeInfo(schema, seenSchemas);
+ case ENUM: return generateEnumTypeInfo(schema);
+ default: throw new AvroSerdeException("Do not yet support: " + schema);
+ }
+ }
+
+ private static TypeInfo generateRecordTypeInfo(Schema schema,
+ Set seenSchemas) throws AvroSerdeException {
+ assert schema.getType().equals(Schema.Type.RECORD);
+
+ if (seenSchemas == null) {
+ seenSchemas = Collections.newSetFromMap(new IdentityHashMap());
+ } else if (seenSchemas.contains(schema)) {
+ throw new AvroSerdeException(
+ "Recursive schemas are not supported. Recursive schema was " + schema
+ .getFullName());
+ }
+ seenSchemas.add(schema);
+
+ List fields = schema.getFields();
+ List fieldNames = new ArrayList(fields.size());
+ List typeInfos = new ArrayList(fields.size());
+
+ for(int i = 0; i < fields.size(); i++) {
+ fieldNames.add(i, fields.get(i).name());
+ typeInfos.add(i, generateTypeInfo(fields.get(i).schema(), seenSchemas));
+ }
+
+ return TypeInfoFactory.getStructTypeInfo(fieldNames, typeInfos);
+ }
+
+ /**
+ * Generate a TypeInfo for an Avro Map. This is made slightly simpler in that
+ * Avro only allows maps with strings for keys.
+ */
+ private static TypeInfo generateMapTypeInfo(Schema schema,
+ Set seenSchemas) throws AvroSerdeException {
+ assert schema.getType().equals(Schema.Type.MAP);
+ Schema valueType = schema.getValueType();
+ TypeInfo ti = generateTypeInfo(valueType, seenSchemas);
+
+ return TypeInfoFactory.getMapTypeInfo(TypeInfoFactory.getPrimitiveTypeInfo("string"), ti);
+ }
+
+ private static TypeInfo generateArrayTypeInfo(Schema schema,
+ Set seenSchemas) throws AvroSerdeException {
+ assert schema.getType().equals(Schema.Type.ARRAY);
+ Schema itemsType = schema.getElementType();
+ TypeInfo itemsTypeInfo = generateTypeInfo(itemsType, seenSchemas);
+
+ return TypeInfoFactory.getListTypeInfo(itemsTypeInfo);
+ }
+
+ private static TypeInfo generateUnionTypeInfo(Schema schema,
+ Set seenSchemas) throws AvroSerdeException {
+ assert schema.getType().equals(Schema.Type.UNION);
+ List types = schema.getTypes();
+
+
+ List typeInfos = new ArrayList(types.size());
+
+ for(Schema type : types) {
+ typeInfos.add(generateTypeInfo(type, seenSchemas));
+ }
+
+ return TypeInfoFactory.getUnionTypeInfo(typeInfos);
+ }
+
+ // Hive doesn't have an Enum type, so we're going to treat them as Strings.
+ // During the deserialize/serialize stage we'll check for enumness and
+ // convert as such.
+ private static TypeInfo generateEnumTypeInfo(Schema schema) {
+ assert schema.getType().equals(Schema.Type.ENUM);
+
+ return TypeInfoFactory.getPrimitiveTypeInfo("string");
+ }
+}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java
new file mode 100644
index 0000000000000000000000000000000000000000..a3a00b3c852c8d5d087905350a09a7b9e92d29c1
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.avro;
+
+import org.apache.avro.Schema;
+import org.apache.hadoop.hive.metastore.schema.utils.AvroSchemaUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.*;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.node.JsonNodeFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * This class contains util methods to convert TypeInfo to Avro schema and vice-versa. This class
+ * is copied from TypeInfoToSchema from Hive source code.
+ */
+public class TypeInfoToSchema {
+
+ private long recordCounter = 0;
+
+ /**
+ * Converts Hive schema to avro schema
+ *
+ * @param columnNames Names of the hive columns
+ * @param columnTypes Hive Column types
+ * @param namespace Namespace of Avro schema
+ * @param name Avro schema name
+ * @param doc Avro schema doc
+ * @return Avro Schema
+ */
+ public Schema convert(List columnNames, List columnTypes,
+ List columnComments, String namespace, String name, String doc) {
+
+ List fields = new ArrayList();
+ for (int i = 0; i < columnNames.size(); ++i) {
+ final String comment = columnComments.size() > i ? columnComments.get(i) : null;
+ final Schema.Field avroField = createAvroField(columnNames.get(i), columnTypes.get(i),
+ comment);
+ fields.addAll(getFields(avroField));
+ }
+
+ if (name == null || name.isEmpty()) {
+ name = "baseRecord";
+ }
+
+ Schema avroSchema = Schema.createRecord(name, doc, namespace, false);
+ avroSchema.setFields(fields);
+ return avroSchema;
+ }
+
+ private Schema.Field createAvroField(String name, TypeInfo typeInfo, String comment) {
+ return new Schema.Field(name, createAvroSchema(typeInfo), comment, null);
+ }
+
+ private Schema createAvroSchema(TypeInfo typeInfo) {
+ Schema schema = null;
+ switch (typeInfo.getCategory()) {
+ case PRIMITIVE:
+ schema = createAvroPrimitive(typeInfo);
+ break;
+ case LIST:
+ schema = createAvroArray(typeInfo);
+ break;
+ case MAP:
+ schema = createAvroMap(typeInfo);
+ break;
+ case STRUCT:
+ schema = createAvroRecord(typeInfo);
+ break;
+ case UNION:
+ schema = createAvroUnion(typeInfo);
+ break;
+ }
+
+ return wrapInUnionWithNull(schema);
+ }
+
+ private Schema createAvroPrimitive(TypeInfo typeInfo) {
+ PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
+ Schema schema;
+ switch (primitiveTypeInfo.getPrimitiveCategory()) {
+ case STRING:
+ schema = Schema.create(Schema.Type.STRING);
+ break;
+ case CHAR:
+ schema = AvroSchemaUtils.getSchemaFor("{" +
+ "\"type\":\"" + AvroSerDeConstants.AVRO_STRING_TYPE_NAME + "\"," +
+ "\"logicalType\":\"" + AvroSerDeConstants.CHAR_TYPE_NAME + "\"," +
+ "\"maxLength\":" + ((CharTypeInfo) typeInfo).getLength() + "}");
+ break;
+ case VARCHAR:
+ schema = AvroSchemaUtils.getSchemaFor("{" +
+ "\"type\":\"" + AvroSerDeConstants.AVRO_STRING_TYPE_NAME + "\"," +
+ "\"logicalType\":\"" + AvroSerDeConstants.VARCHAR_TYPE_NAME + "\"," +
+ "\"maxLength\":" + ((VarcharTypeInfo) typeInfo).getLength() + "}");
+ break;
+ case BINARY:
+ schema = Schema.create(Schema.Type.BYTES);
+ break;
+ case BYTE:
+ schema = Schema.create(Schema.Type.INT);
+ break;
+ case SHORT:
+ schema = Schema.create(Schema.Type.INT);
+ break;
+ case INT:
+ schema = Schema.create(Schema.Type.INT);
+ break;
+ case LONG:
+ schema = Schema.create(Schema.Type.LONG);
+ break;
+ case FLOAT:
+ schema = Schema.create(Schema.Type.FLOAT);
+ break;
+ case DOUBLE:
+ schema = Schema.create(Schema.Type.DOUBLE);
+ break;
+ case BOOLEAN:
+ schema = Schema.create(Schema.Type.BOOLEAN);
+ break;
+ case DECIMAL:
+ DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
+ String precision = String.valueOf(decimalTypeInfo.precision());
+ String scale = String.valueOf(decimalTypeInfo.scale());
+ schema = AvroSchemaUtils.getSchemaFor("{" +
+ "\"type\":\"bytes\"," +
+ "\"logicalType\":\"decimal\"," +
+ "\"precision\":" + precision + "," +
+ "\"scale\":" + scale + "}");
+ break;
+ case DATE:
+ schema = AvroSchemaUtils.getSchemaFor("{" +
+ "\"type\":\"" + AvroSerDeConstants.AVRO_INT_TYPE_NAME + "\"," +
+ "\"logicalType\":\"" + AvroSerDeConstants.DATE_TYPE_NAME + "\"}");
+ break;
+ case TIMESTAMP:
+ schema = AvroSchemaUtils.getSchemaFor("{" +
+ "\"type\":\"" + AvroSerDeConstants.AVRO_LONG_TYPE_NAME + "\"," +
+ "\"logicalType\":\"" + AvroSerDeConstants.TIMESTAMP_TYPE_NAME + "\"}");
+ break;
+ case VOID:
+ schema = Schema.create(Schema.Type.NULL);
+ break;
+ default:
+ throw new UnsupportedOperationException(typeInfo + " is not supported.");
+ }
+ return schema;
+ }
+
+ private Schema createAvroUnion(TypeInfo typeInfo) {
+ List childSchemas = new ArrayList();
+ for (TypeInfo childTypeInfo : ((UnionTypeInfo) typeInfo).getAllUnionObjectTypeInfos()) {
+ final Schema childSchema = createAvroSchema(childTypeInfo);
+ if (childSchema.getType() == Schema.Type.UNION) {
+ childSchemas.addAll(childSchema.getTypes());
+ } else {
+ childSchemas.add(childSchema);
+ }
+ }
+
+ return Schema.createUnion(removeDuplicateNullSchemas(childSchemas));
+ }
+
+ private Schema createAvroRecord(TypeInfo typeInfo) {
+ List childFields = new ArrayList();
+
+ final List allStructFieldNames =
+ ((StructTypeInfo) typeInfo).getAllStructFieldNames();
+ final List allStructFieldTypeInfos =
+ ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos();
+ if (allStructFieldNames.size() != allStructFieldTypeInfos.size()) {
+ throw new IllegalArgumentException("Failed to generate avro schema from hive schema. " +
+ "name and column type differs. names = " + allStructFieldNames + ", types = " +
+ allStructFieldTypeInfos);
+ }
+
+ for (int i = 0; i < allStructFieldNames.size(); ++i) {
+ final TypeInfo childTypeInfo = allStructFieldTypeInfos.get(i);
+ final Schema.Field grandChildSchemaField = createAvroField(allStructFieldNames.get(i),
+ childTypeInfo, childTypeInfo.toString());
+ final List grandChildFields = getFields(grandChildSchemaField);
+ childFields.addAll(grandChildFields);
+ }
+
+ Schema recordSchema = Schema.createRecord("record_" + recordCounter, typeInfo.toString(),
+ null, false);
+ ++recordCounter;
+ recordSchema.setFields(childFields);
+ return recordSchema;
+ }
+
+ private Schema createAvroMap(TypeInfo typeInfo) {
+ TypeInfo keyTypeInfo = ((MapTypeInfo) typeInfo).getMapKeyTypeInfo();
+ if (((PrimitiveTypeInfo) keyTypeInfo).getPrimitiveCategory()
+ != PrimitiveObjectInspector.PrimitiveCategory.STRING) {
+ throw new UnsupportedOperationException("Key of Map can only be a String");
+ }
+
+ TypeInfo valueTypeInfo = ((MapTypeInfo) typeInfo).getMapValueTypeInfo();
+ Schema valueSchema = createAvroSchema(valueTypeInfo);
+
+ return Schema.createMap(valueSchema);
+ }
+
+ private Schema createAvroArray(TypeInfo typeInfo) {
+ ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
+ Schema listSchema = createAvroSchema(listTypeInfo.getListElementTypeInfo());
+ return Schema.createArray(listSchema);
+ }
+
+ private List getFields(Schema.Field schemaField) {
+ List fields = new ArrayList();
+
+ JsonNode nullDefault = JsonNodeFactory.instance.nullNode();
+ if (schemaField.schema().getType() == Schema.Type.RECORD) {
+ for (Schema.Field field : schemaField.schema().getFields()) {
+ fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), nullDefault));
+ }
+ } else {
+ fields.add(new Schema.Field(schemaField.name(), schemaField.schema(), schemaField.doc(),
+ nullDefault));
+ }
+
+ return fields;
+ }
+
+ private Schema wrapInUnionWithNull(Schema schema) {
+ Schema wrappedSchema = schema;
+ switch (schema.getType()) {
+ case NULL:
+ break;
+ case UNION:
+ List existingSchemas = removeDuplicateNullSchemas(schema.getTypes());
+ wrappedSchema = Schema.createUnion(existingSchemas);
+ break;
+ default:
+ wrappedSchema = Schema.createUnion(Arrays.asList(Schema.create(Schema.Type.NULL), schema));
+ }
+
+ return wrappedSchema;
+ }
+
+ private List removeDuplicateNullSchemas(List childSchemas) {
+ List prunedSchemas = new ArrayList();
+ boolean isNullPresent = false;
+ for (Schema schema : childSchemas) {
+ if (schema.getType() == Schema.Type.NULL) {
+ isNullPresent = true;
+ } else {
+ prunedSchemas.add(schema);
+ }
+ }
+ if (isNullPresent) {
+ prunedSchemas.add(0, Schema.create(Schema.Type.NULL));
+ }
+
+ return prunedSchemas;
+ }
+}
\ No newline at end of file
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/schema/reader/TestDefaultStorageSchemaReader.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/schema/reader/TestDefaultStorageSchemaReader.java
new file mode 100644
index 0000000000000000000000000000000000000000..8feb4f2f1a24e94873b8502c9fc410d0b422b415
--- /dev/null
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/schema/reader/TestDefaultStorageSchemaReader.java
@@ -0,0 +1,579 @@
+package org.apache.hadoop.hive.metastore.schema.reader;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.ColumnType;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.metastore.TestHiveMetaStore;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.Type;
+import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.schema.utils.AvroSchemaUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.thrift.TException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class TestDefaultStorageSchemaReader {
+ private static final Logger LOG = LoggerFactory.getLogger(TestDefaultStorageSchemaReader.class);
+ private static final String TEST_DB_NAME = "TEST_DB";
+ private static final String TEST_TABLE_NAME = "TEST_TABLE";
+ private HiveMetaStoreClient client;
+ private Configuration conf;
+ private Warehouse warehouse;
+ private static final int DEFAULT_LIMIT_PARTITION_REQUEST = 100;
+ private static final String AVRO_SERIALIZATION_LIB =
+ "org.apache.hadoop.hive.serde2.avro.AvroSerDe";
+
+ // These schemata are used in other tests
+ static public final String MAP_WITH_PRIMITIVE_VALUE_TYPE = "{\n" +
+ " \"namespace\": \"testing\",\n" +
+ " \"name\": \"oneMap\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"aMap\",\n" +
+ " \"type\":{\"type\":\"map\",\n" +
+ " \"values\":\"long\"}\n" +
+ "\t}\n" +
+ " ]\n" +
+ "}";
+ static public final String ARRAY_WITH_PRIMITIVE_ELEMENT_TYPE = "{\n" +
+ " \"namespace\": \"testing\",\n" +
+ " \"name\": \"oneArray\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"anArray\",\n" +
+ " \"type\":{\"type\":\"array\",\n" +
+ " \"items\":\"string\"}\n" +
+ "\t}\n" +
+ " ]\n" +
+ "}";
+ public static final String RECORD_SCHEMA = "{\n" +
+ " \"namespace\": \"testing.test.mctesty\",\n" +
+ " \"name\": \"oneRecord\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"aRecord\",\n" +
+ " \"type\":{\"type\":\"record\",\n" +
+ " \"name\":\"recordWithinARecord\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"int1\",\n" +
+ " \"type\":\"int\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"boolean1\",\n" +
+ " \"type\":\"boolean\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"long1\",\n" +
+ " \"type\":\"long\"\n" +
+ " }\n" +
+ " ]}\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ public static final String NULLABLE_RECORD_SCHEMA = "[\"null\", " + RECORD_SCHEMA + "]";
+ public static final String UNION_SCHEMA = "{\n" +
+ " \"namespace\": \"test.a.rossa\",\n" +
+ " \"name\": \"oneUnion\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"aUnion\",\n" +
+ " \"type\":[\"int\", \"string\"]\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ public static final String UNION_SCHEMA_2 = "{\n" +
+ " \"namespace\": \"test.a.rossa\",\n" +
+ " \"name\": \"oneUnion\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"aUnion\",\n" +
+ " \"type\":[\"null\", \"int\", \"string\"]\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ public static final String UNION_SCHEMA_3 = "{\n" +
+ " \"namespace\": \"test.a.rossa\",\n" +
+ " \"name\": \"oneUnion\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"aUnion\",\n" +
+ " \"type\":[\"float\",\"int\"]\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ public static final String UNION_SCHEMA_4 = "{\n" +
+ " \"namespace\": \"test.a.rossa\",\n" +
+ " \"name\": \"oneUnion\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"aUnion\",\n" +
+ " \"type\":[\"int\",\"float\",\"long\"]\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ public static final String ENUM_SCHEMA = "{\n" +
+ " \"namespace\": \"clever.namespace.name.in.space\",\n" +
+ " \"name\": \"oneEnum\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"baddies\",\n" +
+ " \"type\":{\"type\":\"enum\",\"name\":\"villians\", \"symbols\": " +
+ "[\"DALEKS\", \"CYBERMEN\", \"SLITHEEN\", \"JAGRAFESS\"]}\n" +
+ " \n" +
+ " \n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ public static final String FIXED_SCHEMA = "{\n" +
+ " \"namespace\": \"ecapseman\",\n" +
+ " \"name\": \"oneFixed\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"hash\",\n" +
+ " \"type\":{\"type\": \"fixed\", \"name\": \"MD5\", \"size\": 16}\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ public static final String NULLABLE_STRING_SCHEMA = "{\n" +
+ " \"type\": \"record\", \n" +
+ " \"name\": \"nullableUnionTest\",\n" +
+ " \"fields\" : [\n" +
+ " {\"name\":\"nullableString\", \"type\":[\"null\", \"string\"]}\n" +
+ " ]\n" +
+ "}";
+ public static final String MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA = "{\n" +
+ " \"namespace\": \"testing\",\n" +
+ " \"name\": \"mapWithNullableUnionTest\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"aMap\",\n" +
+ " \"type\":{\"type\":\"map\",\n" +
+ " \"values\":[\"null\",\"long\"]}\n" +
+ "\t}\n" +
+ " ]\n" +
+ "}";
+ public static final String NULLABLE_ENUM_SCHEMA = "{\n" +
+ " \"namespace\": \"clever.namespace.name.in.space\",\n" +
+ " \"name\": \"nullableUnionTest\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"nullableEnum\",\n" +
+ " \"type\": [\"null\", {\"type\":\"enum\",\"name\":\"villians\", \"symbols\": " +
+ "[\"DALEKS\", \"CYBERMEN\", \"SLITHEEN\", \"JAGRAFESS\"]}]\n" +
+ " \n" +
+ " \n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+ public static final String BYTES_SCHEMA = "{\n" +
+ " \"type\": \"record\", \n" +
+ " \"name\": \"bytesTest\",\n" +
+ " \"fields\" : [\n" +
+ " {\"name\":\"bytesField\", \"type\":\"bytes\"}\n" +
+ " ]\n" +
+ "}";
+
+ public static final String KITCHEN_SINK_SCHEMA = "{\n" +
+ " \"namespace\": \"org.apache.hadoop.hive\",\n" +
+ " \"name\": \"kitchsink\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"string1\",\n" +
+ " \"type\":\"string\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"string2\",\n" +
+ " \"type\":\"string\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"int1\",\n" +
+ " \"type\":\"int\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"boolean1\",\n" +
+ " \"type\":\"boolean\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"long1\",\n" +
+ " \"type\":\"long\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"float1\",\n" +
+ " \"type\":\"float\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"double1\",\n" +
+ " \"type\":\"double\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"inner_record1\",\n" +
+ " \"type\":{ \"type\":\"record\",\n" +
+ " \"name\":\"inner_record1_impl\",\n" +
+ " \"fields\": [\n" +
+ " {\"name\":\"int_in_inner_record1\",\n" +
+ " \"type\":\"int\"},\n" +
+ " {\"name\":\"string_in_inner_record1\",\n" +
+ " \"type\":\"string\"}\n" +
+ " ]\n" +
+ " }\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"enum1\",\n" +
+ " \"type\":{\"type\":\"enum\", \"name\":\"enum1_values\", " +
+ "\"symbols\":[\"ENUM1_VALUES_VALUE1\",\"ENUM1_VALUES_VALUE2\", \"ENUM1_VALUES_VALUE3\"]}\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"array1\",\n" +
+ " \"type\":{\"type\":\"array\", \"items\":\"string\"}\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"map1\",\n" +
+ " \"type\":{\"type\":\"map\", \"values\":\"string\"}\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"union1\",\n" +
+ " \"type\":[\"float\", \"boolean\", \"string\"]\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"fixed1\",\n" +
+ " \"type\":{\"type\":\"fixed\", \"name\":\"fourbytes\", \"size\":4}\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"null1\",\n" +
+ " \"type\":\"null\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"UnionNullInt\",\n" +
+ " \"type\":[\"int\", \"null\"]\n" +
+ " },\n" +
+ " {\n" +
+ " \"name\":\"bytes1\",\n" +
+ " \"type\":\"bytes\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ @Before
+ public void setUp() throws Exception {
+ conf = MetastoreConf.newMetastoreConf();
+ warehouse = new Warehouse(conf);
+
+ // set some values to use for getting conf. vars
+ MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.METRICS_ENABLED, true);
+ conf.set("hive.key1", "value1");
+ conf.set("hive.key2", "http://www.example.com");
+ conf.set("hive.key3", "");
+ conf.set("hive.key4", "0");
+ conf.set("datanucleus.autoCreateTables", "false");
+
+ MetaStoreTestUtils.setConfForStandloneMode(conf);
+ MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX, 2);
+ MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.LIMIT_PARTITION_REQUEST,
+ DEFAULT_LIMIT_PARTITION_REQUEST);
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL,
+ DefaultStorageSchemaReader.class.getName());
+ client = createClient();
+ }
+
+ @After
+ public void closeClient() {
+ client.close();
+ }
+
+ private void silentDropDatabase(String dbName) throws TException {
+ try {
+ for (String tableName : client.getTables(dbName, "*")) {
+ client.dropTable(dbName, tableName);
+ }
+ client.dropDatabase(dbName);
+ } catch (NoSuchObjectException | InvalidOperationException e) {
+ // NOP
+ }
+ }
+
+ private HiveMetaStoreClient createClient() throws Exception {
+ try {
+ return new HiveMetaStoreClient(conf);
+ } catch (Throwable e) {
+ System.err.println("Unable to open the metastore");
+ System.err.println(StringUtils.stringifyException(e));
+ throw new Exception(e);
+ }
+ }
+
+ @Test
+ public void testSimpleAvroTable() throws TException, IOException {
+ List fields = new ArrayList<>(2);
+ FieldSchema field = new FieldSchema();
+ field.setName("name");
+ field.setType("string");
+ field.setComment("Test name comment");
+ fields.add(field);
+
+ field = new FieldSchema();
+ field.setName("age");
+ field.setType("int");
+ field.setComment("Test age comment");
+ fields.add(field);
+
+ createTable(TEST_DB_NAME, TEST_TABLE_NAME, AVRO_SERIALIZATION_LIB, fields, null);
+ List retFields = client.getFields("testdb", "testTbl");
+ verifyTableFields(fields, retFields, null);
+ }
+
+ private Table createTable(String dbName, String tblName, String serializationLib,
+ List fields, Map tblProperties) throws TException, IOException {
+ client.dropTable(dbName, tblName);
+ silentDropDatabase(dbName);
+ Database db = new Database();
+ db.setName(dbName);
+ client.createDatabase(db);
+ db = client.getDatabase(dbName);
+ Path dbPath = new Path(db.getLocationUri());
+ FileSystem fs = FileSystem.get(dbPath.toUri(), conf);
+ String typeName = "dummy";
+ client.dropType(typeName);
+ Type typ1 = new Type();
+ typ1.setName(typeName);
+ typ1.setFields(fields);
+ client.createType(typ1);
+
+ Table tbl = new TableBuilder().setDbName(dbName).setTableName(tblName).setCols(typ1.getFields())
+ .setSerdeLib(serializationLib).setTableParams(tblProperties).build();
+ client.createTable(tbl);
+ return client.getTable(dbName, tblName);
+ }
+
+ @Test
+ public void testExternalSchemaAvroTable() throws TException, IOException {
+ //map
+ createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, MAP_WITH_PRIMITIVE_VALUE_TYPE);
+ List retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME);
+ Assert.assertEquals("Unexpected number of fields", 1, retFields.size());
+ Assert.assertEquals("Unexpected name of the field", "aMap", retFields.get(0).getName());
+ Assert.assertEquals("Unexpected type of the field", "map",
+ retFields.get(0).getType());
+ Assert.assertEquals("Unexpected comment of the field", "", retFields.get(0).getComment());
+
+ //list
+ createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME,
+ ARRAY_WITH_PRIMITIVE_ELEMENT_TYPE);
+ retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME);
+ Assert.assertEquals("Unexpected number of fields", 1, retFields.size());
+ Assert.assertEquals("Unexpected name of the field", "anArray", retFields.get(0).getName());
+ Assert
+ .assertEquals("Unexpected type of the field", "array", retFields.get(0).getType());
+
+ //struct
+ createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, RECORD_SCHEMA);
+ retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME);
+ Assert.assertEquals("Unexpected number of fields", 1, retFields.size());
+ Assert.assertEquals("Unexpected name of the field", "aRecord", retFields.get(0).getName());
+ Assert.assertEquals("Unexpected type of the field",
+ "struct", retFields.get(0).getType());
+
+ //union
+ createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, UNION_SCHEMA);
+ retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME);
+ Assert.assertEquals("Unexpected number of fields", 1, retFields.size());
+ Assert.assertEquals("Unexpected name of the field", "aUnion", retFields.get(0).getName());
+ Assert.assertEquals("Unexpected type of the field", "uniontype",
+ retFields.get(0).getType());
+
+ //union-2
+ createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, UNION_SCHEMA_2);
+ retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME);
+ Assert.assertEquals("Unexpected number of fields", 1, retFields.size());
+ Assert.assertEquals("Unexpected name of the field", "aUnion", retFields.get(0).getName());
+ Assert.assertEquals("Unexpected type of the field", "uniontype",
+ retFields.get(0).getType());
+
+ //union_3
+ createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, UNION_SCHEMA_3);
+ retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME);
+ Assert.assertEquals("Unexpected number of fields", 1, retFields.size());
+ Assert.assertEquals("Unexpected name of the field", "aUnion", retFields.get(0).getName());
+ Assert.assertEquals("Unexpected type of the field", "uniontype",
+ retFields.get(0).getType());
+
+ //union_4
+ createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, UNION_SCHEMA_4);
+ retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME);
+ Assert.assertEquals("Unexpected number of fields", 1, retFields.size());
+ Assert.assertEquals("Unexpected name of the field", "aUnion", retFields.get(0).getName());
+ Assert.assertEquals("Unexpected type of the field", "uniontype",
+ retFields.get(0).getType());
+
+ //enum
+ // Enums are one of two Avro types that Hive doesn't have any native support for.
+ // Column names - we lose the enumness of this schema
+ // Column types become string
+ createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, ENUM_SCHEMA);
+ retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME);
+ Assert.assertEquals("Unexpected number of fields", 1, retFields.size());
+ Assert.assertEquals("Unexpected name of the field", "baddies", retFields.get(0).getName());
+ Assert.assertEquals("Unexpected type of the field", "string",
+ retFields.get(0).getType());
+
+ // Hive has no concept of Avro's fixed type. Fixed -> arrays of bytes
+ createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, FIXED_SCHEMA);
+ retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME);
+ Assert.assertEquals("Unexpected number of fields", 1, retFields.size());
+ Assert.assertEquals("Unexpected name of the field", "hash", retFields.get(0).getName());
+ Assert.assertEquals("Unexpected type of the field", "binary",
+ retFields.get(0).getType());
+
+ //nullable string
+ createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, NULLABLE_STRING_SCHEMA);
+ retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME);
+ Assert.assertEquals("Unexpected number of fields", 1, retFields.size());
+ Assert.assertEquals("Unexpected name of the field", "nullableString", retFields.get(0).getName());
+ Assert.assertEquals("Unexpected type of the field", "string",
+ retFields.get(0).getType());
+
+ //map with nullable value - That Union[T, NULL] is converted to just T, within a Map
+ createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA);
+ retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME);
+ Assert.assertEquals("Unexpected number of fields", 1, retFields.size());
+ Assert.assertEquals("Unexpected name of the field", "aMap", retFields.get(0).getName());
+ Assert.assertEquals("Unexpected type of the field", "map",
+ retFields.get(0).getType());
+
+ // That Union[T, NULL] is converted to just T.
+ createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, NULLABLE_ENUM_SCHEMA);
+ retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME);
+ Assert.assertEquals("Unexpected number of fields", 1, retFields.size());
+ Assert.assertEquals("Unexpected name of the field", "nullableEnum", retFields.get(0).getName());
+ Assert.assertEquals("Unexpected type of the field", "string",
+ retFields.get(0).getType());
+
+ createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, BYTES_SCHEMA);
+ retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME);
+ Assert.assertEquals("Unexpected number of fields", 1, retFields.size());
+ Assert.assertEquals("Unexpected name of the field", "bytesField", retFields.get(0).getName());
+ Assert.assertEquals("Unexpected type of the field", "binary",
+ retFields.get(0).getType());
+
+ createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, KITCHEN_SINK_SCHEMA);
+ retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME);
+ Assert.assertEquals("Unexpected number of fields", 16, retFields.size());
+ //There are 16 fields in this schema. Instead of verifying all we verify the interesting ones
+ //(ones which have not been tested above)
+ Assert
+ .assertEquals("Unexpected name of 8th field", "inner_record1", retFields.get(7).getName());
+ Assert.assertEquals("Unexpected type of the field",
+ "struct",
+ retFields.get(7).getType());
+
+ Assert.assertEquals("Unexpected field name of the 10th field", "array1",
+ retFields.get(9).getName());
+ Assert.assertEquals("Unexpected field type of the 10th field", "array",
+ retFields.get(9).getType());
+
+ Assert.assertEquals("Unexpected field name of the 11th field", "map1",
+ retFields.get(10).getName());
+ Assert.assertEquals("Unexpected field type of the 11th field", "map",
+ retFields.get(10).getType());
+
+ Assert.assertEquals("Unexpected field name of the 12th field", "union1",
+ retFields.get(11).getName());
+ Assert
+ .assertEquals("Unexpected field type of the 12th field", "uniontype",
+ retFields.get(11).getType());
+
+ Assert.assertEquals("Unexpected field name of the 14th field", "null1",
+ retFields.get(13).getName());
+ Assert.assertEquals("Unexpected field type of the 14th field", "void",
+ retFields.get(13).getType());
+
+ Assert.assertEquals("Unexpected field name of the 15th field", "UnionNullInt",
+ retFields.get(14).getName());
+ Assert.assertEquals("Unexpected field type of the 15th field", "int",
+ retFields.get(14).getType());
+ }
+
+ private void createAvroTableWithExternalSchema(String dbName, String tblName, String schemaStr)
+ throws TException, IOException {
+ List fields = new ArrayList<>(0);
+ Map tblParams = new HashMap<>();
+ tblParams.put(AvroSchemaUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName(), schemaStr);
+ createTable(dbName, tblName, AVRO_SERIALIZATION_LIB, fields, tblParams);
+ }
+
+ @Test
+ public void testSimpleTable() throws TException, IOException {
+ List fields = new ArrayList<>(2);
+ FieldSchema field = new FieldSchema();
+ field.setName("name");
+ field.setType("string");
+ field.setComment("Test name comment");
+ fields.add(field);
+
+ field = new FieldSchema();
+ field.setName("age");
+ field.setType("int");
+ field.setComment("Test age comment");
+ fields.add(field);
+
+ createTable(TEST_DB_NAME, TEST_TABLE_NAME, null, fields, null);
+ List retFields = client.getFields("testDb", "testTbl");
+ verifyTableFields(fields, retFields, null);
+ }
+
+ private void verifyTableFields(List expected, List actual,
+ String nullCommentText) {
+ Assert.assertEquals(expected.size(), actual.size());
+ int size = expected.size();
+ for (int i = 0; i < size; i++) {
+ FieldSchema expectedField = expected.get(i);
+ FieldSchema actualField = actual.get(i);
+ Assert.assertEquals("Name does not match for field " + (i + 1), expectedField.getName(),
+ actualField.getName());
+ Assert.assertEquals("Type does not match for field " + (i + 1), expectedField.getType(),
+ actualField.getType());
+ String expectedComment = null;
+ if (expectedField.getComment() == null && nullCommentText != null) {
+ expectedComment = nullCommentText;
+ } else {
+ expectedComment = expectedField.getComment();
+ }
+ Assert.assertEquals("Comment does not match for field " + (i + 1), expectedComment,
+ actualField.getComment());
+ }
+ }
+}
diff --git a/storage-api/hive-storage-api/pom.xml b/storage-api/hive-storage-api/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..f6680687576d28969dfad983a81f432625d945fe
--- /dev/null
+++ b/storage-api/hive-storage-api/pom.xml
@@ -0,0 +1,133 @@
+
+
+
+ 4.0.0
+
+ org.apache.hive
+ storage-api
+ 3.0.0-SNAPSHOT
+ ../pom.xml
+
+
+ org.apache.hive
+ hive-storage-api
+ 3.0.0-SNAPSHOT
+ jar
+ Hive Storage API
+
+
+ 2.6
+ 1.1.3
+ 14.0.1
+ 2.8.1
+ 4.11
+ 1.7.10
+ 3.0.0
+
+
+
+
+
+ commons-lang
+ commons-lang
+ ${commons-lang.version}
+
+
+ org.apache.hadoop
+ hadoop-annotations
+ ${hadoop.version}
+
+
+ org.apache.hadoop
+ hadoop-common
+ ${hadoop.version}
+ provided
+
+
+ com.google.guava
+ guava
+
+
+ com.google.code.findbugs
+ jsr305
+
+
+ commons-logging
+ commons-logging
+
+
+ javax.servlet
+ servlet-api
+
+
+ javax.servlet.jsp
+ jsp-api
+
+
+ org.apache.avro
+ avro
+
+
+ org.apache.httpcomponents
+ httpcore
+
+
+ org.apache.httpcomponents
+ httpclient
+
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ org.mortbay.jetty
+ jetty
+
+
+ org.mortbay.jetty
+ jetty-util
+
+
+
+
+ org.slf4j
+ slf4j-api
+ ${slf4j.version}
+
+
+
+
+ com.google.guava
+ guava
+ ${guava.version}
+ test
+
+
+ junit
+ junit
+ ${junit.version}
+ test
+
+
+ commons-logging
+ commons-logging
+ ${commons-logging.version}
+ test
+
+
+
+
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
new file mode 100644
index 0000000000000000000000000000000000000000..e5025bf1407aaf51910849d70e4ccda08a70964d
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.common.io.DiskRange;
+
+/**
+ * Disk range information class containing disk ranges and total length.
+ */
+public class DiskRangeInfo {
+ List diskRanges; // TODO: use DiskRangeList instead
+ long totalLength;
+
+ public DiskRangeInfo(int indexBaseOffset) {
+ this.diskRanges = new ArrayList<>();
+ // Some data is missing from the stream for PPD uncompressed read (because index offset is
+ // relative to the entire stream and we only read part of stream if RGs are filtered; unlike
+ // with compressed data where PPD only filters CBs, so we always get full CB, and index offset
+ // is relative to CB). To take care of the case when UncompressedStream goes seeking around by
+ // its incorrect (relative to partial stream) index offset, we will increase the length by our
+ // offset-relative-to-the-stream, and also account for it in buffers (see createDiskRangeInfo).
+ // So, index offset now works; as long as noone seeks into this data before the RG (why would
+ // they), everything works. This is hacky... Stream shouldn't depend on having all the data.
+ this.totalLength = indexBaseOffset;
+ }
+
+ public void addDiskRange(DiskRange diskRange) {
+ diskRanges.add(diskRange);
+ totalLength += diskRange.getLength();
+ }
+
+ public List getDiskRanges() {
+ return diskRanges;
+ }
+
+ public long getTotalLength() {
+ return totalLength;
+ }
+}
+
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/Pool.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/Pool.java
new file mode 100644
index 0000000000000000000000000000000000000000..5ab4ee7802fc385ff04e0f4fcd54398ed31073c5
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/Pool.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common;
+
+/** Simple object pool to prevent GC on small objects passed between threads. */
+public interface Pool {
+ /** Object helper for objects stored in the pool. */
+ public interface PoolObjectHelper {
+ /** Called to create an object when one cannot be provided.
+ * @return a newly allocated object
+ */
+ T create();
+ /** Called before the object is put in the pool (regardless of whether put succeeds).
+ * @param t the object to reset
+ */
+ void resetBeforeOffer(T t);
+ }
+
+ T take();
+ void offer(T t);
+ int size();
+}
\ No newline at end of file
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
new file mode 100644
index 0000000000000000000000000000000000000000..94b8c58d28219c13f045f6d9e8dd093a4b3607d9
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+import java.util.Arrays;
+import java.util.BitSet;
+
+/**
+ * An implementation of {@link org.apache.hadoop.hive.common.ValidTxnList} for use by the compactor.
+ *
+ * Compaction should only include txns up to smallest open txn (exclussive).
+ * There may be aborted txns in the snapshot represented by this ValidCompactorTxnList.
+ * Thus {@link #isTxnRangeValid(long, long)} returns NONE for any range that inluces any unresolved
+ * transactions. Any txn above {@code highWatermark} is unresolved.
+ * These produce the logic we need to assure that the compactor only sees records less than the lowest
+ * open transaction when choosing which files to compact, but that it still ignores aborted
+ * records when compacting.
+ *
+ * See org.apache.hadoop.hive.metastore.txn.TxnUtils#createValidCompactTxnList() for proper
+ * way to construct this.
+ */
+public class ValidCompactorTxnList extends ValidReadTxnList {
+ public ValidCompactorTxnList() {
+ super();
+ }
+ public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark) {
+ this(abortedTxnList, abortedBits, highWatermark, Long.MAX_VALUE);
+ }
+ /**
+ * @param abortedTxnList list of all aborted transactions
+ * @param abortedBits bitset marking whether the corresponding transaction is aborted
+ * @param highWatermark highest committed transaction to be considered for compaction,
+ * equivalently (lowest_open_txn - 1).
+ */
+ public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark, long minOpenTxnId) {
+ // abortedBits should be all true as everything in exceptions are aborted txns
+ super(abortedTxnList, abortedBits, highWatermark, minOpenTxnId);
+ if(this.exceptions.length <= 0) {
+ return;
+ }
+ //now that exceptions (aka abortedTxnList) is sorted
+ int idx = Arrays.binarySearch(this.exceptions, highWatermark);
+ int lastElementPos;
+ if(idx < 0) {
+ int insertionPoint = -idx - 1 ;//see Arrays.binarySearch() JavaDoc
+ lastElementPos = insertionPoint - 1;
+ }
+ else {
+ lastElementPos = idx;
+ }
+ /*
+ * ensure that we throw out any exceptions above highWatermark to make
+ * {@link #isTxnValid(long)} faster
+ */
+ this.exceptions = Arrays.copyOf(this.exceptions, lastElementPos + 1);
+ }
+ public ValidCompactorTxnList(String value) {
+ super(value);
+ }
+ /**
+ * Returns org.apache.hadoop.hive.common.ValidTxnList.RangeResponse.ALL if all txns in
+ * the range are resolved and RangeResponse.NONE otherwise
+ */
+ @Override
+ public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId) {
+ return highWatermark >= maxTxnId ? RangeResponse.ALL : RangeResponse.NONE;
+ }
+
+ @Override
+ public boolean isTxnAborted(long txnid) {
+ return Arrays.binarySearch(exceptions, txnid) >= 0;
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
new file mode 100644
index 0000000000000000000000000000000000000000..ccdd4b727a59b106da0def2d0173c917e1c5c0f7
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+import java.util.Arrays;
+import java.util.BitSet;
+
+/**
+ * An implementation of {@link org.apache.hadoop.hive.common.ValidTxnList} for use by readers.
+ * This class will view a transaction as valid only if it is committed. Both open and aborted
+ * transactions will be seen as invalid.
+ */
+public class ValidReadTxnList implements ValidTxnList {
+
+ protected long[] exceptions;
+ protected BitSet abortedBits; // BitSet for flagging aborted transactions. Bit is true if aborted, false if open
+ //default value means there are no open txn in the snapshot
+ private long minOpenTxn = Long.MAX_VALUE;
+ protected long highWatermark;
+
+ public ValidReadTxnList() {
+ this(new long[0], new BitSet(), Long.MAX_VALUE, Long.MAX_VALUE);
+ }
+
+ /**
+ * Used if there are no open transactions in the snapshot
+ */
+ public ValidReadTxnList(long[] exceptions, BitSet abortedBits, long highWatermark) {
+ this(exceptions, abortedBits, highWatermark, Long.MAX_VALUE);
+ }
+ public ValidReadTxnList(long[] exceptions, BitSet abortedBits, long highWatermark, long minOpenTxn) {
+ if (exceptions.length > 0) {
+ this.minOpenTxn = minOpenTxn;
+ }
+ this.exceptions = exceptions;
+ this.abortedBits = abortedBits;
+ this.highWatermark = highWatermark;
+ }
+
+ public ValidReadTxnList(String value) {
+ readFromString(value);
+ }
+
+ @Override
+ public boolean isTxnValid(long txnid) {
+ if (highWatermark < txnid) {
+ return false;
+ }
+ return Arrays.binarySearch(exceptions, txnid) < 0;
+ }
+
+ /**
+ * We cannot use a base file if its range contains an open txn.
+ * @param txnid from base_xxxx
+ */
+ @Override
+ public boolean isValidBase(long txnid) {
+ return minOpenTxn > txnid && txnid <= highWatermark;
+ }
+ @Override
+ public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId) {
+ // check the easy cases first
+ if (highWatermark < minTxnId) {
+ return RangeResponse.NONE;
+ } else if (exceptions.length > 0 && exceptions[0] > maxTxnId) {
+ return RangeResponse.ALL;
+ }
+
+ // since the exceptions and the range in question overlap, count the
+ // exceptions in the range
+ long count = Math.max(0, maxTxnId - highWatermark);
+ for(long txn: exceptions) {
+ if (minTxnId <= txn && txn <= maxTxnId) {
+ count += 1;
+ }
+ }
+
+ if (count == 0) {
+ return RangeResponse.ALL;
+ } else if (count == (maxTxnId - minTxnId + 1)) {
+ return RangeResponse.NONE;
+ } else {
+ return RangeResponse.SOME;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return writeToString();
+ }
+
+ @Override
+ public String writeToString() {
+ StringBuilder buf = new StringBuilder();
+ buf.append(highWatermark);
+ buf.append(':');
+ buf.append(minOpenTxn);
+ if (exceptions.length == 0) {
+ buf.append(':'); // separator for open txns
+ buf.append(':'); // separator for aborted txns
+ } else {
+ StringBuilder open = new StringBuilder();
+ StringBuilder abort = new StringBuilder();
+ for (int i = 0; i < exceptions.length; i++) {
+ if (abortedBits.get(i)) {
+ if (abort.length() > 0) {
+ abort.append(',');
+ }
+ abort.append(exceptions[i]);
+ } else {
+ if (open.length() > 0) {
+ open.append(',');
+ }
+ open.append(exceptions[i]);
+ }
+ }
+ buf.append(':');
+ buf.append(open);
+ buf.append(':');
+ buf.append(abort);
+ }
+ return buf.toString();
+ }
+
+ @Override
+ public void readFromString(String src) {
+ if (src == null || src.length() == 0) {
+ highWatermark = Long.MAX_VALUE;
+ exceptions = new long[0];
+ abortedBits = new BitSet();
+ } else {
+ String[] values = src.split(":");
+ highWatermark = Long.parseLong(values[0]);
+ minOpenTxn = Long.parseLong(values[1]);
+ String[] openTxns = new String[0];
+ String[] abortedTxns = new String[0];
+ if (values.length < 3) {
+ openTxns = new String[0];
+ abortedTxns = new String[0];
+ } else if (values.length == 3) {
+ if (!values[2].isEmpty()) {
+ openTxns = values[2].split(",");
+ }
+ } else {
+ if (!values[2].isEmpty()) {
+ openTxns = values[2].split(",");
+ }
+ if (!values[3].isEmpty()) {
+ abortedTxns = values[3].split(",");
+ }
+ }
+ exceptions = new long[openTxns.length + abortedTxns.length];
+ int i = 0;
+ for (String open : openTxns) {
+ exceptions[i++] = Long.parseLong(open);
+ }
+ for (String abort : abortedTxns) {
+ exceptions[i++] = Long.parseLong(abort);
+ }
+ Arrays.sort(exceptions);
+ abortedBits = new BitSet(exceptions.length);
+ for (String abort : abortedTxns) {
+ int index = Arrays.binarySearch(exceptions, Long.parseLong(abort));
+ abortedBits.set(index);
+ }
+ }
+ }
+
+ @Override
+ public long getHighWatermark() {
+ return highWatermark;
+ }
+
+ @Override
+ public long[] getInvalidTransactions() {
+ return exceptions;
+ }
+
+ @Override
+ public Long getMinOpenTxn() {
+ return minOpenTxn == Long.MAX_VALUE ? null : minOpenTxn;
+ }
+
+ @Override
+ public boolean isTxnAborted(long txnid) {
+ int index = Arrays.binarySearch(exceptions, txnid);
+ return index >= 0 && abortedBits.get(index);
+ }
+
+ @Override
+ public RangeResponse isTxnRangeAborted(long minTxnId, long maxTxnId) {
+ // check the easy cases first
+ if (highWatermark < minTxnId) {
+ return RangeResponse.NONE;
+ }
+
+ int count = 0; // number of aborted txns found in exceptions
+
+ // traverse the aborted txns list, starting at first aborted txn index
+ for (int i = abortedBits.nextSetBit(0); i >= 0; i = abortedBits.nextSetBit(i + 1)) {
+ long abortedTxnId = exceptions[i];
+ if (abortedTxnId > maxTxnId) { // we've already gone beyond the specified range
+ break;
+ }
+ if (abortedTxnId >= minTxnId && abortedTxnId <= maxTxnId) {
+ count++;
+ }
+ }
+
+ if (count == 0) {
+ return RangeResponse.NONE;
+ } else if (count == (maxTxnId - minTxnId + 1)) {
+ return RangeResponse.ALL;
+ } else {
+ return RangeResponse.SOME;
+ }
+ }
+}
+
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
new file mode 100644
index 0000000000000000000000000000000000000000..108e5ca85cfab26403e74adefd7f349d4105ee6b
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+/**
+ * Models the list of transactions that should be included in a snapshot.
+ * It is modelled as a high water mark, which is the largest transaction id that
+ * has been committed and a list of transactions that are not included.
+ */
+public interface ValidTxnList {
+
+ /**
+ * Key used to store valid txn list in a
+ * {@link org.apache.hadoop.conf.Configuration} object.
+ */
+ public static final String VALID_TXNS_KEY = "hive.txn.valid.txns";
+
+ /**
+ * The response to a range query. NONE means no values in this range match,
+ * SOME mean that some do, and ALL means that every value does.
+ */
+ public enum RangeResponse {NONE, SOME, ALL};
+
+ /**
+ * Indicates whether a given transaction is valid. Note that valid may have different meanings
+ * for different implementations, as some will only want to see committed transactions and some
+ * both committed and aborted.
+ * @param txnid id for the transaction
+ * @return true if valid, false otherwise
+ */
+ public boolean isTxnValid(long txnid);
+
+ /**
+ * Returns {@code true} if such base file can be used to materialize the snapshot represented by
+ * this {@code ValidTxnList}.
+ * @param txnid highest txn in a given base_xxxx file
+ */
+ public boolean isValidBase(long txnid);
+
+ /**
+ * Find out if a range of transaction ids are valid. Note that valid may have different meanings
+ * for different implementations, as some will only want to see committed transactions and some
+ * both committed and aborted.
+ * @param minTxnId minimum txnid to look for, inclusive
+ * @param maxTxnId maximum txnid to look for, inclusive
+ * @return Indicate whether none, some, or all of these transactions are valid.
+ */
+ public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId);
+
+ /**
+ * Write this validTxnList into a string. This should produce a string that
+ * can be used by {@link #readFromString(String)} to populate a validTxnsList.
+ */
+ public String writeToString();
+
+ /**
+ * Populate this validTxnList from the string. It is assumed that the string
+ * was created via {@link #writeToString()} and the exceptions list is sorted.
+ * @param src source string.
+ */
+ public void readFromString(String src);
+
+ /**
+ * Get the largest transaction id used.
+ * @return largest transaction id used
+ */
+ public long getHighWatermark();
+
+ /**
+ * Get the list of transactions under the high water mark that are not valid. Note that invalid
+ * may have different meanings for different implementations, as some will only want to see open
+ * transactions and some both open and aborted.
+ * @return a list of invalid transaction ids
+ */
+ public long[] getInvalidTransactions();
+
+ /**
+ * Indicates whether a given transaction is aborted.
+ * @param txnid id for the transaction
+ * @return true if aborted, false otherwise
+ */
+ public boolean isTxnAborted(long txnid);
+
+ /**
+ * Find out if a range of transaction ids are aborted.
+ * @param minTxnId minimum txnid to look for, inclusive
+ * @param maxTxnId maximum txnid to look for, inclusive
+ * @return Indicate whether none, some, or all of these transactions are aborted.
+ */
+ public RangeResponse isTxnRangeAborted(long minTxnId, long maxTxnId);
+
+ /**
+ * Returns smallest Open transaction in this set, {@code null} if there is none.
+ */
+ Long getMinOpenTxn();
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java
new file mode 100644
index 0000000000000000000000000000000000000000..775233cb6e3cd091b570634a7e030e4352f24ea6
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.io;
+
+import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
+
+/** An allocator provided externally to storage classes to allocate MemoryBuffer-s. */
+public interface Allocator {
+ public static class AllocatorOutOfMemoryException extends RuntimeException {
+ public AllocatorOutOfMemoryException(String msg) {
+ super(msg);
+ }
+
+ private static final long serialVersionUID = 268124648177151761L;
+ }
+
+ public interface BufferObjectFactory {
+ MemoryBuffer create();
+ }
+
+ /**
+ * Allocates multiple buffers of a given size.
+ * @param dest Array where buffers are placed. Objects are reused if already there
+ * (see createUnallocated), created otherwise.
+ * @param size Allocation size.
+ * @throws AllocatorOutOfMemoryException Cannot allocate.
+ */
+ @Deprecated
+ void allocateMultiple(MemoryBuffer[] dest, int size) throws AllocatorOutOfMemoryException;
+
+ /**
+ * Allocates multiple buffers of a given size.
+ * @param dest Array where buffers are placed. Objects are reused if already there
+ * (see createUnallocated), created otherwise.
+ * @param size Allocation size.
+ * @param factory A factory to create the objects in the dest array, if needed.
+ * @throws AllocatorOutOfMemoryException Cannot allocate.
+ */
+ void allocateMultiple(MemoryBuffer[] dest, int size, BufferObjectFactory factory)
+ throws AllocatorOutOfMemoryException;
+
+ /**
+ * Creates an unallocated memory buffer object. This object can be passed to allocateMultiple
+ * to allocate; this is useful if data structures are created for separate buffers that can
+ * later be allocated together.
+ * @return a new unallocated memory buffer
+ */
+ @Deprecated
+ MemoryBuffer createUnallocated();
+
+ /** Deallocates a memory buffer.
+ * @param buffer the buffer to deallocate
+ */
+ void deallocate(MemoryBuffer buffer);
+
+ /** Whether the allocator uses direct buffers.
+ * @return are they direct buffers?
+ */
+ boolean isDirectAlloc();
+
+ /** Maximum allocation size supported by this allocator.
+ * @return a number of bytes
+ */
+ int getMaxAllocation();
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..552f20e276a6cd2d93ecc09292985c842ee22b87
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.io;
+
+import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
+
+/** An abstract data cache that IO formats can use to retrieve and cache data. */
+public interface DataCache {
+ public static final class BooleanRef {
+ public boolean value;
+ }
+
+ /** Disk range factory used during cache retrieval. */
+ public interface DiskRangeListFactory {
+ DiskRangeList createCacheChunk(MemoryBuffer buffer, long startOffset, long endOffset);
+ }
+
+ /**
+ * Gets file data for particular offsets. The range list is modified in place; it is then
+ * returned (since the list head could have changed). Ranges are replaced with cached ranges.
+ *
+ * Any such buffer is locked in cache to prevent eviction, and must therefore be released
+ * back to cache via a corresponding call (releaseBuffer) when the caller is done with it.
+ *
+ * In case of partial overlap with cached data, full cache blocks are always returned;
+ * there's no capacity for partial matches in return type. The rules are as follows:
+ * 1) If the requested range starts in the middle of a cached range, that cached range will not
+ * be returned by default (e.g. if [100,200) and [200,300) are cached, the request for
+ * [150,300) will only return [200,300) from cache). This may be configurable in impls.
+ * This is because we assume well-known range start offsets are used (rg/stripe offsets), so
+ * a request from the middle of the start doesn't make sense.
+ * 2) If the requested range ends in the middle of a cached range, that entire cached range will
+ * be returned (e.g. if [100,200) and [200,300) are cached, the request for [100,250) will
+ * return both ranges). It should really be same as #1, however currently ORC uses estimated
+ * end offsets; if we don't return the end block, the caller may read it from disk needlessly.
+ *
+ * @param fileKey Unique ID of the target file on the file system.
+ * @param range A set of DiskRange-s (linked list) that is to be retrieved. May be modified.
+ * @param baseOffset base offset for the ranges (stripe/stream offset in case of ORC).
+ * @param factory A factory to produce DiskRangeList-s out of cached MemoryBuffer-s.
+ * @param gotAllData An out param - whether all the requested data was found in cache.
+ * @return The new or modified list of DiskRange-s, where some ranges may contain cached data.
+ */
+ DiskRangeList getFileData(Object fileKey, DiskRangeList range, long baseOffset,
+ DiskRangeListFactory factory, BooleanRef gotAllData);
+
+ /**
+ * Puts file data into cache, or gets older data in case of collisions.
+ *
+ * The memory buffers provided MUST be allocated via an allocator returned by getAllocator
+ * method, to allow cache implementations that evict and then de-allocate the buffer.
+ *
+ * It is assumed that the caller will use the data immediately, therefore any buffers provided
+ * to putFileData (or returned due to cache collision) are locked in cache to prevent eviction,
+ * and must therefore be released back to cache via a corresponding call (releaseBuffer) when the
+ * caller is done with it. Buffers rejected due to conflict will neither be locked, nor
+ * automatically deallocated. The caller must take care to discard these buffers.
+ *
+ * @param fileKey Unique ID of the target file on the file system.
+ * @param ranges The ranges for which the data is being cached. These objects will not be stored.
+ * @param data The data for the corresponding ranges.
+ * @param baseOffset base offset for the ranges (stripe/stream offset in case of ORC).
+ * @return null if all data was put; bitmask indicating which chunks were not put otherwise;
+ * the replacement chunks from cache are updated directly in the array.
+ */
+ long[] putFileData(Object fileKey, DiskRange[] ranges, MemoryBuffer[] data, long baseOffset);
+
+ /**
+ * Releases the buffer returned by getFileData/provided to putFileData back to cache.
+ * See respective javadocs for details.
+ * @param buffer the buffer to release
+ */
+ void releaseBuffer(MemoryBuffer buffer);
+
+ /**
+ * Notifies the cache that the buffer returned from getFileData/provided to putFileData will
+ * be used by another consumer and therefore released multiple times (one more time per call).
+ * @param buffer the buffer to reuse
+ */
+ void reuseBuffer(MemoryBuffer buffer);
+
+ /**
+ * Gets the allocator associated with this DataCache.
+ * @return the allocator
+ */
+ Allocator getAllocator();
+
+ /**
+ * Gets the buffer object factory associated with this DataCache, to use with allocator.
+ * @return the factory
+ */
+ Allocator.BufferObjectFactory getDataBufferFactory();
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRange.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRange.java
new file mode 100644
index 0000000000000000000000000000000000000000..33aecf53e703b72482e4e29c19170bbdebc815e3
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRange.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.io;
+
+import java.nio.ByteBuffer;
+
+/**
+ * The sections of a file.
+ */
+public class DiskRange {
+ /** The first address. */
+ protected long offset;
+ /** The address afterwards. */
+ protected long end;
+
+ public DiskRange(long offset, long end) {
+ this.offset = offset;
+ this.end = end;
+ if (end < offset) {
+ throw new IllegalArgumentException("invalid range " + this);
+ }
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+ return equalRange((DiskRange) other);
+ }
+
+ public boolean equalRange(DiskRange other) {
+ return other.offset == offset && other.end == end;
+ }
+
+ @Override
+ public int hashCode() {
+ return (int)(offset ^ (offset >>> 32)) * 31 + (int)(end ^ (end >>> 32));
+ }
+
+ @Override
+ public String toString() {
+ return "range start: " + offset + " end: " + end;
+ }
+
+ public long getOffset() {
+ return offset;
+ }
+
+ public long getEnd() {
+ return end;
+ }
+
+ public int getLength() {
+ long len = this.end - this.offset;
+ assert len <= Integer.MAX_VALUE;
+ return (int)len;
+ }
+
+ // For subclasses
+ public boolean hasData() {
+ return false;
+ }
+
+ public DiskRange sliceAndShift(long offset, long end, long shiftBy) {
+ // Rather, unexpected usage exception.
+ throw new UnsupportedOperationException();
+ }
+
+ public ByteBuffer getData() {
+ throw new UnsupportedOperationException();
+ }
+
+ protected boolean merge(long otherOffset, long otherEnd) {
+ if (!overlap(offset, end, otherOffset, otherEnd)) return false;
+ offset = Math.min(offset, otherOffset);
+ end = Math.max(end, otherEnd);
+ return true;
+ }
+
+ private static boolean overlap(long leftA, long rightA, long leftB, long rightB) {
+ if (leftA <= leftB) {
+ return rightA >= leftB;
+ }
+ return rightB >= leftA;
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
new file mode 100644
index 0000000000000000000000000000000000000000..13494b6e413aff1ba4c5a97bd3afa96e39d148a7
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
@@ -0,0 +1,297 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.io;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Java linked list iterator interface is convoluted, and moreover concurrent modifications
+ * of the same list by multiple iterators are impossible. Hence, this.
+ * Java also doesn't support multiple inheritance, so this cannot be done as "aspect"... */
+public class DiskRangeList extends DiskRange {
+ private static final Logger LOG = LoggerFactory.getLogger(DiskRangeList.class);
+ public DiskRangeList prev, next;
+
+ public DiskRangeList(long offset, long end) {
+ super(offset, end);
+ }
+
+ /** Replaces this element with another in the list; returns the new element.
+ * @param other the disk range to swap into this list
+ * @return the new element
+ */
+ public DiskRangeList replaceSelfWith(DiskRangeList other) {
+ checkArg(other);
+ other.prev = this.prev;
+ other.next = this.next;
+ if (this.prev != null) {
+ checkOrder(this.prev, other, this);
+ this.prev.next = other;
+ }
+ if (this.next != null) {
+ checkOrder(other, this.next, this);
+ this.next.prev = other;
+ }
+ this.next = this.prev = null;
+ return other;
+ }
+
+ private final static void checkOrder(DiskRangeList prev, DiskRangeList next, DiskRangeList ref) {
+ if (prev.getEnd() <= next.getOffset()) return;
+ assertInvalidOrder(ref.prev == null ? ref : ref.prev, prev, next);
+ }
+
+ private final static void assertInvalidOrder(
+ DiskRangeList ref, DiskRangeList prev, DiskRangeList next) {
+ String error = "Elements not in order " + prev + " and " + next
+ + "; trying to insert into " + stringifyDiskRanges(ref);
+ LOG.error(error);
+ throw new AssertionError(error);
+ }
+
+ // TODO: this duplicates a method in ORC, but the method should actually be here.
+ public final static String stringifyDiskRanges(DiskRangeList range) {
+ StringBuilder buffer = new StringBuilder();
+ buffer.append("[");
+ boolean isFirst = true;
+ while (range != null) {
+ if (!isFirst) {
+ buffer.append(", {");
+ } else {
+ buffer.append("{");
+ }
+ isFirst = false;
+ buffer.append(range.toString());
+ buffer.append("}");
+ range = range.next;
+ }
+ buffer.append("]");
+ return buffer.toString();
+ }
+
+ private void checkArg(DiskRangeList other) {
+ if (other == this) {
+ // The only case where duplicate elements matter... the others are handled by the below.
+ throw new AssertionError("Inserting self into the list [" + other + "]");
+ }
+ if (other.prev != null || other.next != null) {
+ throw new AssertionError("[" + other + "] is part of another list; prev ["
+ + other.prev + "], next [" + other.next + "]");
+ }
+ }
+
+ /**
+ * Inserts an intersecting range before current in the list and adjusts offset accordingly.
+ * @param other the element to insert
+ * @return the new element.
+ */
+ public DiskRangeList insertPartBefore(DiskRangeList other) {
+ checkArg(other);
+ if (other.end <= this.offset || other.end > this.end) {
+ assertInvalidOrder(this.prev == null ? this : this.prev, other, this);
+ }
+ this.offset = other.end;
+ other.prev = this.prev;
+ other.next = this;
+ if (this.prev != null) {
+ checkOrder(this.prev, other, this.prev);
+ this.prev.next = other;
+ }
+ this.prev = other;
+ return other;
+ }
+
+ /**
+ * Inserts an element after current in the list.
+ * @param other the new element to insert
+ * @return the new element.
+ * */
+ public DiskRangeList insertAfter(DiskRangeList other) {
+ checkArg(other);
+ checkOrder(this, other, this);
+ return insertAfterInternal(other);
+ }
+
+ private DiskRangeList insertAfterInternal(DiskRangeList other) {
+ other.next = this.next;
+ other.prev = this;
+ if (this.next != null) {
+ checkOrder(other, this.next, this);
+ this.next.prev = other;
+ }
+ this.next = other;
+ return other;
+ }
+
+ /**
+ * Inserts an intersecting range after current in the list and adjusts offset accordingly.
+ * @param other the new element to insert
+ * @return the new element.
+ */
+ public DiskRangeList insertPartAfter(DiskRangeList other) {
+ // The only allowed non-overlapping option is extra bytes at the end.
+ if (other.offset > this.end || other.offset <= this.offset || other.end <= this.offset) {
+ assertInvalidOrder(this.prev == null ? this : this.prev, this, other);
+ }
+ this.end = other.offset;
+ return insertAfter(other);
+ }
+
+ /** Removes an element after current from the list. */
+ public void removeAfter() {
+ DiskRangeList other = this.next;
+ if (this == other) {
+ throw new AssertionError("Invalid duplicate [" + other + "]");
+ }
+ this.next = other.next;
+ if (this.next != null) {
+ this.next.prev = this;
+ }
+ other.next = other.prev = null;
+ }
+
+ /** Removes the current element from the list. */
+ public void removeSelf() {
+ if (this.prev == this || this.next == this) {
+ throw new AssertionError("Invalid duplicate [" + this + "]");
+ }
+ if (this.prev != null) {
+ this.prev.next = this.next;
+ }
+ if (this.next != null) {
+ this.next.prev = this.prev;
+ }
+ this.next = this.prev = null;
+ }
+
+ /** Splits current element in the list, using DiskRange::slice.
+ * @param cOffset the position to split the list
+ * @return the split list
+ */
+ public final DiskRangeList split(long cOffset) {
+ DiskRangeList right = insertAfterInternal((DiskRangeList)this.sliceAndShift(cOffset, end, 0));
+ DiskRangeList left = replaceSelfWith((DiskRangeList)this.sliceAndShift(offset, cOffset, 0));
+ checkOrder(left, right, left); // Prev/next are already checked in the calls.
+ return left;
+ }
+
+ public boolean hasContiguousNext() {
+ return next != null && end == next.offset;
+ }
+
+ // @VisibleForTesting
+ public int listSize() {
+ int result = 1;
+ DiskRangeList current = this.next;
+ while (current != null) {
+ ++result;
+ current = current.next;
+ }
+ return result;
+ }
+
+ public long getTotalLength() {
+ long totalLength = getLength();
+ DiskRangeList current = next;
+ while (current != null) {
+ totalLength += current.getLength();
+ current = current.next;
+ }
+ return totalLength;
+ }
+
+ // @VisibleForTesting
+ public DiskRangeList[] listToArray() {
+ DiskRangeList[] result = new DiskRangeList[listSize()];
+ int i = 0;
+ DiskRangeList current = this.next;
+ while (current != null) {
+ result[i] = current;
+ ++i;
+ current = current.next;
+ }
+ return result;
+ }
+
+ public static class CreateHelper {
+ private DiskRangeList tail = null, head;
+
+ public DiskRangeList getTail() {
+ return tail;
+ }
+
+ public void addOrMerge(long offset, long end, boolean doMerge, boolean doLogNew) {
+ if (doMerge && tail != null && tail.merge(offset, end)) return;
+ if (doLogNew) {
+ LOG.debug("Creating new range; last range (which can include some previous adds) was "
+ + tail);
+ }
+ DiskRangeList node = new DiskRangeList(offset, end);
+ if (tail == null) {
+ head = tail = node;
+ } else {
+ tail = tail.insertAfter(node);
+ }
+ }
+
+ public DiskRangeList get() {
+ return head;
+ }
+
+ public DiskRangeList extract() {
+ DiskRangeList result = head;
+ head = null;
+ return result;
+ }
+ }
+
+ /**
+ * List in-place mutation helper - a bogus first element that is inserted before list head,
+ * and thus remains constant even if head is replaced with some new range via in-place list
+ * mutation. extract() can be used to obtain the modified list.
+ */
+ public static class MutateHelper extends DiskRangeList {
+ public MutateHelper(DiskRangeList head) {
+ super(-1, -1);
+ assert head != null;
+ assert head.prev == null;
+ this.next = head;
+ head.prev = this;
+ }
+
+ public DiskRangeList get() {
+ return next;
+ }
+
+ public DiskRangeList extract() {
+ DiskRangeList result = this.next;
+ assert result != null;
+ this.next = result.prev = null;
+ return result;
+ }
+ }
+
+ public void setEnd(long newEnd) {
+ assert newEnd >= this.offset;
+ assert this.next == null || this.next.offset >= newEnd;
+ this.end = newEnd;
+ if (this.next != null) {
+ checkOrder(this, this.next, this);
+ }
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java
new file mode 100644
index 0000000000000000000000000000000000000000..e684ece3edbf420e05adadc7a1f9599553ceb6d4
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.io;
+
+
+import java.nio.ByteBuffer;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.hadoop.hive.common.io.encoded.MemoryBufferOrBuffers;
+
+public interface FileMetadataCache {
+ /**
+ * @return Metadata for a given file (ORC or Parquet footer).
+ * The caller must decref this buffer when done.
+ */
+ MemoryBufferOrBuffers getFileMetadata(Object fileKey);
+
+ /**
+ * Puts the metadata for a given file (e.g. a footer buffer into cache).
+ * @param fileKey The file key.
+ * @param length The footer length.
+ * @param is The stream to read the footer from.
+ * @return The buffer or buffers representing the cached footer.
+ * The caller must decref this buffer when done.
+ */
+ MemoryBufferOrBuffers putFileMetadata(
+ Object fileKey, int length, InputStream is) throws IOException;
+
+ MemoryBufferOrBuffers putFileMetadata(Object fileKey, ByteBuffer tailBuffer);
+
+ /**
+ * Releases the buffer returned from getFileMetadata or putFileMetadata method.
+ * @param buffer The buffer to release.
+ */
+ void decRefBuffer(MemoryBufferOrBuffers buffer);
+}
\ No newline at end of file
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java
new file mode 100644
index 0000000000000000000000000000000000000000..63084b902a0cc684865f18a74621110480138643
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.io.encoded;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A block of data for a given section of a file, similar to VRB but in encoded form.
+ * Stores a set of buffers for each encoded stream that is a part of each column.
+ */
+public class EncodedColumnBatch {
+ /**
+ * Slice of the data for a stream for some column, stored inside MemoryBuffer's.
+ * ColumnStreamData can be reused for many EncodedColumnBatch-es (e.g. dictionary stream), so
+ * it tracks the number of such users via a refcount.
+ */
+ public static class ColumnStreamData {
+ private List cacheBuffers;
+ /** Base offset from the beginning of the indexable unit; for example, for ORC,
+ * offset from the CB in a compressed file, from the stream in uncompressed file. */
+ private int indexBaseOffset = 0;
+
+ /** Reference count. */
+ private AtomicInteger refCount = new AtomicInteger(0);
+
+ public void reset() {
+ cacheBuffers.clear();
+ refCount.set(0);
+ indexBaseOffset = 0;
+ }
+
+ public void incRef() {
+ refCount.incrementAndGet();
+ }
+
+ public int decRef() {
+ int i = refCount.decrementAndGet();
+ assert i >= 0;
+ return i;
+ }
+
+ public List getCacheBuffers() {
+ return cacheBuffers;
+ }
+
+ public void setCacheBuffers(List cacheBuffers) {
+ this.cacheBuffers = cacheBuffers;
+ }
+
+ public int getIndexBaseOffset() {
+ return indexBaseOffset;
+ }
+
+ public void setIndexBaseOffset(int indexBaseOffset) {
+ this.indexBaseOffset = indexBaseOffset;
+ }
+
+ @Override
+ public String toString() {
+ String bufStr = "";
+ if (cacheBuffers != null) {
+ for (MemoryBuffer mb : cacheBuffers) {
+ bufStr += mb.getClass().getSimpleName() + " with " + mb.getByteBufferRaw().remaining() + " bytes, ";
+ }
+ }
+ return "ColumnStreamData [cacheBuffers=[" + bufStr
+ + "], indexBaseOffset=" + indexBaseOffset + "]";
+ }
+
+ }
+
+ /** The key that is used to map this batch to source location. */
+ protected BatchKey batchKey;
+ /**
+ * Stream data for each column that has true in the corresponding hasData position.
+ * For each column, streams are indexed by kind (for ORC), with missing elements being null.
+ */
+ protected ColumnStreamData[][] columnData;
+ /** Indicates which columns have data. Correspond to columnData elements. */
+ protected boolean[] hasData;
+
+ public void reset() {
+ if (hasData != null) {
+ Arrays.fill(hasData, false);
+ }
+ if (columnData == null) return;
+ for (int i = 0; i < columnData.length; ++i) {
+ if (columnData[i] == null) continue;
+ for (int j = 0; j < columnData[i].length; ++j) {
+ columnData[i][j] = null;
+ }
+ }
+ }
+
+
+ public void initColumn(int colIx, int streamCount) {
+ hasData[colIx] = true;
+ if (columnData[colIx] == null || columnData[colIx].length != streamCount) {
+ columnData[colIx] = new ColumnStreamData[streamCount];
+ }
+ }
+
+ private static final Logger LOG = LoggerFactory.getLogger(EncodedColumnBatch.class);
+ public void setStreamData(int colIx, int streamIx, ColumnStreamData csd) {
+ assert hasData[colIx];
+ columnData[colIx][streamIx] = csd;
+ }
+
+ public BatchKey getBatchKey() {
+ return batchKey;
+ }
+
+ public ColumnStreamData[] getColumnData(int colIx) {
+ if (!hasData[colIx]) throw new AssertionError("No data for column " + colIx);
+ return columnData[colIx];
+ }
+
+ public int getTotalColCount() {
+ return columnData.length; // Includes the columns that have no data
+ }
+
+ protected void resetColumnArrays(int columnCount) {
+ if (hasData != null && columnCount == hasData.length) {
+ Arrays.fill(hasData, false);
+ return;
+ }
+ hasData = new boolean[columnCount];
+ ColumnStreamData[][] columnData = new ColumnStreamData[columnCount][];
+ if (this.columnData != null) {
+ for (int i = 0; i < Math.min(columnData.length, this.columnData.length); ++i) {
+ columnData[i] = this.columnData[i];
+ }
+ }
+ this.columnData = columnData;
+ }
+
+ public boolean hasData(int colIx) {
+ return hasData[colIx];
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java
new file mode 100644
index 0000000000000000000000000000000000000000..201c5e2bc4ee36b1249e687429f22ced9d77c94c
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.io.encoded;
+
+import java.nio.ByteBuffer;
+
+/** Abstract interface for any class wrapping a ByteBuffer. */
+public interface MemoryBuffer {
+ /**
+ * Get the raw byte buffer that backs this buffer.
+ * Note - raw buffer should not be modified.
+ * @return the shared byte buffer
+ */
+ public ByteBuffer getByteBufferRaw();
+ public ByteBuffer getByteBufferDup();
+}
\ No newline at end of file
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBufferOrBuffers.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBufferOrBuffers.java
new file mode 100644
index 0000000000000000000000000000000000000000..c04310a2ba9390a29e86673002e199edff37ba91
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBufferOrBuffers.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.io.encoded;
+
+public interface MemoryBufferOrBuffers {
+ MemoryBuffer getSingleBuffer();
+ MemoryBuffer[] getMultipleBuffers();
+}
\ No newline at end of file
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/DataTypePhysicalVariation.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/DataTypePhysicalVariation.java
new file mode 100644
index 0000000000000000000000000000000000000000..778c8c3d793092780b56101ca0054502d7a8f5cc
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/DataTypePhysicalVariation.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.type;
+
+public enum DataTypePhysicalVariation {
+ NONE,
+ DECIMAL_64
+}
\ No newline at end of file
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java
new file mode 100644
index 0000000000000000000000000000000000000000..4484ed27b5a432f96451db0f94a6553074a6dd2e
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java
@@ -0,0 +1,759 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.type;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+/**
+ * FastHiveDecimal is a mutable fast decimal object. It is the base class for both the
+ * HiveDecimal and HiveDecimalWritable classes. All fast* methods are protected so they
+ * cannot be accessed by clients of HiveDecimal and HiveDecimalWritable. HiveDecimal ensures
+ * it creates new objects when the value changes since it provides immutable semantics;
+ * HiveDecimalWritable does not create new objects since it provides mutable semantics.
+ *
+ * The methods in this class are shells that pickup the member variables from FastHiveDecimal
+ * parameters and pass them as individual parameters to static methods in the FastHiveDecimalImpl
+ * class that do the real work.
+ *
+ * NOTE: The rationale for fast decimal is in FastHiveDecimalImpl.
+ */
+public class FastHiveDecimal {
+
+ /*
+ * We use protected for the fields so the FastHiveDecimalImpl class can access them. Other
+ * classes including HiveDecimal should not access these fields directly.
+ */
+
+ // See FastHiveDecimalImpl for more details on these fields.
+
+ // -1 when negative; 0 when decimal is zero; 1 when positive.
+ protected int fastSignum;
+
+ // Decimal longwords.
+ protected long fast2;
+ protected long fast1;
+ protected long fast0;
+
+ // The number of integer digits in the decimal. When the integer portion is zero, this is 0.
+ protected int fastIntegerDigitCount;
+
+ // The scale of the decimal.
+ protected int fastScale;
+
+ // Used for legacy HiveDecimalV1 setScale compatibility for binary / display serialization of
+ // trailing zeroes (or rounding).
+ protected int fastSerializationScale;
+
+ protected FastHiveDecimal() {
+ fastReset();
+ }
+
+ protected FastHiveDecimal(FastHiveDecimal fastDec) {
+ this();
+ fastSignum = fastDec.fastSignum;
+ fast0 = fastDec.fast0;
+ fast1 = fastDec.fast1;
+ fast2 = fastDec.fast2;
+ fastIntegerDigitCount = fastDec.fastIntegerDigitCount;
+ fastScale = fastDec.fastScale;
+
+ // Not propagated.
+ fastSerializationScale = -1;
+ }
+
+ protected FastHiveDecimal(int fastSignum, FastHiveDecimal fastDec) {
+ this();
+ this.fastSignum = fastSignum;
+ fast0 = fastDec.fast0;
+ fast1 = fastDec.fast1;
+ fast2 = fastDec.fast2;
+ fastIntegerDigitCount = fastDec.fastIntegerDigitCount;
+ fastScale = fastDec.fastScale;
+
+ // Not propagated.
+ fastSerializationScale = -1;
+ }
+
+ protected FastHiveDecimal(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+ this();
+ this.fastSignum = fastSignum;
+ this.fast0 = fast0;
+ this.fast1 = fast1;
+ this.fast2 = fast2;
+ this.fastIntegerDigitCount = fastIntegerDigitCount;
+ this.fastScale = fastScale;
+
+ fastSerializationScale = -1;
+ }
+
+ protected FastHiveDecimal(long longValue) {
+ this();
+ FastHiveDecimalImpl.fastSetFromLong(longValue, this);
+ }
+
+ protected FastHiveDecimal(String string) {
+ this();
+ FastHiveDecimalImpl.fastSetFromString(string, false, this);
+ }
+
+ protected void fastReset() {
+ fastSignum = 0;
+ fast0 = 0;
+ fast1 = 0;
+ fast2 = 0;
+ fastIntegerDigitCount = 0;
+ fastScale = 0;
+ fastSerializationScale = -1;
+ }
+
+ protected void fastSet(FastHiveDecimal fastDec) {
+ fastSignum = fastDec.fastSignum;
+ fast0 = fastDec.fast0;
+ fast1 = fastDec.fast1;
+ fast2 = fastDec.fast2;
+ fastIntegerDigitCount = fastDec.fastIntegerDigitCount;
+ fastScale = fastDec.fastScale;
+ fastSerializationScale = fastDec.fastSerializationScale;
+ }
+
+ protected void fastSet(
+ int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) {
+ this.fastSignum = fastSignum;
+ this.fast0 = fast0;
+ this.fast1 = fast1;
+ this.fast2 = fast2;
+ this.fastIntegerDigitCount = fastIntegerDigitCount;
+ this.fastScale = fastScale;
+
+ // Not specified.
+ fastSerializationScale = -1;
+ }
+
+ protected void fastSetSerializationScale(int fastSerializationScale) {
+ this.fastSerializationScale = fastSerializationScale;
+ }
+
+ protected int fastSerializationScale() {
+ return fastSerializationScale;
+ }
+
+ protected static final String STRING_ENFORCE_PRECISION_OUT_OF_RANGE =
+ "Decimal precision out of allowed range [1," + HiveDecimal.MAX_PRECISION + "]";
+ protected static final String STRING_ENFORCE_SCALE_OUT_OF_RANGE =
+ "Decimal scale out of allowed range [0," + HiveDecimal.MAX_SCALE + "]";
+ protected static final String STRING_ENFORCE_SCALE_LESS_THAN_EQUAL_PRECISION =
+ "Decimal scale must be less than or equal to precision";
+
+ protected boolean fastSetFromBigDecimal(
+ BigDecimal bigDecimal, boolean allowRounding) {
+ return
+ FastHiveDecimalImpl.fastSetFromBigDecimal(
+ bigDecimal, allowRounding, this);
+ }
+
+ protected boolean fastSetFromBigInteger(
+ BigInteger bigInteger) {
+ return
+ FastHiveDecimalImpl.fastSetFromBigInteger(
+ bigInteger, this);
+ }
+
+ protected boolean fastSetFromBigIntegerAndScale(
+ BigInteger bigInteger, int scale) {
+ return
+ FastHiveDecimalImpl.fastSetFromBigInteger(
+ bigInteger, scale, this);
+ }
+
+ protected boolean fastSetFromString(String string, boolean trimBlanks) {
+ byte[] bytes = string.getBytes();
+ return
+ fastSetFromBytes(
+ bytes, 0, bytes.length, trimBlanks);
+ }
+
+ protected boolean fastSetFromBytes(byte[] bytes, int offset, int length, boolean trimBlanks) {
+ return
+ FastHiveDecimalImpl.fastSetFromBytes(
+ bytes, offset, length, trimBlanks, this);
+ }
+
+ protected boolean fastSetFromDigitsOnlyBytesAndScale(
+ boolean isNegative, byte[] bytes, int offset, int length, int scale) {
+ return
+ FastHiveDecimalImpl.fastSetFromDigitsOnlyBytesAndScale(
+ isNegative, bytes, offset, length, scale, this);
+ }
+
+ protected void fastSetFromInt(int intValue) {
+ FastHiveDecimalImpl.fastSetFromInt(intValue, this);
+ }
+
+ protected void fastSetFromLong(long longValue) {
+ FastHiveDecimalImpl.fastSetFromLong(longValue, this);
+ }
+
+ protected boolean fastSetFromLongAndScale(long longValue, int scale) {
+ return
+ FastHiveDecimalImpl.fastSetFromLongAndScale(
+ longValue, scale, this);
+ }
+
+ protected boolean fastSetFromFloat(float floatValue) {
+ return
+ FastHiveDecimalImpl.fastSetFromFloat(
+ floatValue, this);
+ }
+
+ protected boolean fastSetFromDouble(double doubleValue) {
+ return
+ FastHiveDecimalImpl.fastSetFromDouble(
+ doubleValue, this);
+ }
+
+ protected void fastFractionPortion() {
+ FastHiveDecimalImpl.fastFractionPortion(
+ fastSignum, fast0, fast1, fast2, fastScale,
+ this);
+ }
+
+ protected void fastIntegerPortion() {
+ FastHiveDecimalImpl.fastIntegerPortion(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale,
+ this);
+ }
+
+ protected static final int FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ = 8 * 3;
+
+ protected boolean fastSerializationUtilsRead(
+ InputStream inputStream, int scale,
+ byte[] scratchBytes) throws IOException, EOFException {
+ return
+ FastHiveDecimalImpl.fastSerializationUtilsRead(
+ inputStream, scale, scratchBytes, this);
+ }
+
+ protected boolean fastSetFromBigIntegerBytesAndScale(
+ byte[] bytes, int offset, int length, int scale) {
+ return FastHiveDecimalImpl.fastSetFromBigIntegerBytesAndScale(
+ bytes, offset, length, scale, this);
+ }
+
+ protected static final int SCRATCH_LONGS_LEN_FAST_SERIALIZATION_UTILS_WRITE = 6;
+
+ protected boolean fastSerializationUtilsWrite(OutputStream outputStream,
+ long[] scratchLongs)
+ throws IOException {
+ return
+ FastHiveDecimalImpl.fastSerializationUtilsWrite(
+ outputStream,
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale,
+ scratchLongs);
+ }
+
+ /*
+ * Deserializes 64-bit decimals up to the maximum 64-bit precision (18 decimal digits).
+ */
+ protected void fastDeserialize64(long decimalLong, int scale) {
+ FastHiveDecimalImpl.fastDeserialize64(
+ decimalLong, scale, this);
+ }
+
+ /*
+ * Serializes decimal64 up to the maximum 64-bit precision (18 decimal digits).
+ */
+ protected long fastSerialize64(int scale) {
+ return
+ FastHiveDecimalImpl.fastSerialize64(
+ scale,
+ fastSignum, fast1, fast0, fastScale);
+ }
+
+ // The fastBigIntegerBytes method returns 3 56 bit (7 byte) words and a possible sign byte.
+ // However, the fastBigIntegerBytes can take on trailing zeroes -- so make it larger.
+ protected static final int FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES = 1 + 48;
+ protected static final int FAST_SCRATCH_LONGS_LEN = 6;
+
+ protected int fastBigIntegerBytes(
+ long[] scratchLongs, byte[] buffer) {
+ return
+ FastHiveDecimalImpl.fastBigIntegerBytes(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ fastSerializationScale,
+ scratchLongs, buffer);
+ }
+
+ protected int fastBigIntegerBytesScaled(
+ int serializationScale,
+ long[] scratchLongs, byte[] buffer) {
+ return
+ FastHiveDecimalImpl.fastBigIntegerBytesScaled(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ serializationScale,
+ scratchLongs, buffer);
+ }
+
+ protected boolean fastIsByte() {
+ return
+ FastHiveDecimalImpl.fastIsByte(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ }
+
+ protected byte fastByteValueClip() {
+ return
+ FastHiveDecimalImpl.fastByteValueClip(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ }
+
+ protected boolean fastIsShort() {
+ return
+ FastHiveDecimalImpl.fastIsShort(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ }
+
+ protected short fastShortValueClip() {
+ return
+ FastHiveDecimalImpl.fastShortValueClip(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ }
+
+ protected boolean fastIsInt() {
+ return
+ FastHiveDecimalImpl.fastIsInt(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ }
+
+ protected int fastIntValueClip() {
+ return
+ FastHiveDecimalImpl.fastIntValueClip(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ }
+
+ protected boolean fastIsLong() {
+ return
+ FastHiveDecimalImpl.fastIsLong(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ }
+
+ protected long fastLongValueClip() {
+ return
+ FastHiveDecimalImpl.fastLongValueClip(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ }
+
+ protected float fastFloatValue() {
+ return
+ FastHiveDecimalImpl.fastFloatValue(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ }
+
+ protected double fastDoubleValue() {
+ return
+ FastHiveDecimalImpl.fastDoubleValue(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ }
+
+ protected BigInteger fastBigIntegerValue() {
+ return
+ FastHiveDecimalImpl.fastBigIntegerValue(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ fastSerializationScale);
+ }
+
+ protected BigDecimal fastBigDecimalValue() {
+ return
+ FastHiveDecimalImpl.fastBigDecimalValue(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale);
+ }
+
+ protected int fastScale() {
+ return fastScale;
+ }
+
+ protected int fastSignum() {
+ return fastSignum;
+ }
+
+ protected int fastCompareTo(FastHiveDecimal right) {
+ return
+ FastHiveDecimalImpl.fastCompareTo(
+ fastSignum, fast0, fast1, fast2,
+ fastScale,
+ right.fastSignum, right.fast0, right.fast1, right.fast2,
+ right.fastScale);
+ }
+
+ protected static int fastCompareTo(FastHiveDecimal left, FastHiveDecimal right) {
+ return
+ FastHiveDecimalImpl.fastCompareTo(
+ left.fastSignum, left.fast0, left.fast1, left.fast2,
+ left.fastScale,
+ right.fastSignum, right.fast0, right.fast1, right.fast2,
+ right.fastScale);
+ }
+
+ protected boolean fastEquals(FastHiveDecimal that) {
+ return
+ FastHiveDecimalImpl.fastEquals(
+ fastSignum, fast0, fast1, fast2,
+ fastScale,
+ that.fastSignum, that.fast0, that.fast1, that.fast2,
+ that.fastScale);
+ }
+
+ protected void fastAbs() {
+ fastSignum = 1;
+ }
+
+ protected void fastNegate() {
+ if (fastSignum == 0) {
+ return;
+ }
+ fastSignum = (fastSignum == 1 ? -1 : 1);
+ }
+
+ protected int fastNewFasterHashCode() {
+ return
+ FastHiveDecimalImpl.fastNewFasterHashCode(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ }
+
+ protected int fastHashCode() {
+ return
+ FastHiveDecimalImpl.fastHashCode(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ }
+
+ protected int fastIntegerDigitCount() {
+ return fastIntegerDigitCount;
+ }
+
+ protected int fastSqlPrecision() {
+ return
+ FastHiveDecimalImpl.fastSqlPrecision(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale);
+ }
+
+ protected int fastRawPrecision() {
+ return
+ FastHiveDecimalImpl.fastRawPrecision(
+ fastSignum, fast0, fast1, fast2);
+ }
+
+ protected boolean fastScaleByPowerOfTen(
+ int n,
+ FastHiveDecimal fastResult) {
+ return
+ FastHiveDecimalImpl.fastScaleByPowerOfTen(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale,
+ n,
+ fastResult);
+ }
+
+ protected static String fastRoundingModeToString(int roundingMode) {
+ String roundingModeString;
+ switch (roundingMode) {
+ case BigDecimal.ROUND_DOWN:
+ roundingModeString = "ROUND_DOWN";
+ break;
+ case BigDecimal.ROUND_UP:
+ roundingModeString = "ROUND_UP";
+ break;
+ case BigDecimal.ROUND_FLOOR:
+ roundingModeString = "ROUND_FLOOR";
+ break;
+ case BigDecimal.ROUND_CEILING:
+ roundingModeString = "ROUND_CEILING";
+ break;
+ case BigDecimal.ROUND_HALF_UP:
+ roundingModeString = "ROUND_HALF_UP";
+ break;
+ case BigDecimal.ROUND_HALF_EVEN:
+ roundingModeString = "ROUND_HALF_EVEN";
+ break;
+ default:
+ roundingModeString = "Unknown";
+ }
+ return roundingModeString + " (" + roundingMode + ")";
+ }
+
+ protected boolean fastRound(
+ int newScale, int roundingMode,
+ FastHiveDecimal fastResult) {
+ return
+ FastHiveDecimalImpl.fastRound(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ newScale, roundingMode,
+ fastResult);
+ }
+
+ protected boolean isAllZeroesBelow(
+ int power) {
+ return
+ FastHiveDecimalImpl.isAllZeroesBelow(
+ fastSignum, fast0, fast1, fast2, power);
+ }
+
+ protected boolean fastEnforcePrecisionScale(
+ int maxPrecision, int maxScale) {
+ if (maxPrecision <= 0 || maxPrecision > HiveDecimal.MAX_PRECISION) {
+ return false;
+ }
+ if (maxScale < 0 || maxScale > HiveDecimal.MAX_SCALE) {
+ return false;
+ }
+ /*
+ if (!fastIsValid()) {
+ fastRaiseInvalidException();
+ }
+ */
+ FastCheckPrecisionScaleStatus status =
+ FastHiveDecimalImpl.fastCheckPrecisionScale(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ maxPrecision, maxScale);
+ switch (status) {
+ case NO_CHANGE:
+ return true;
+ case OVERFLOW:
+ return false;
+ case UPDATE_SCALE_DOWN:
+ {
+ if (!FastHiveDecimalImpl.fastUpdatePrecisionScale(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ maxPrecision, maxScale, status,
+ this)) {
+ return false;
+ }
+ /*
+ if (!fastIsValid()) {
+ fastRaiseInvalidException();
+ }
+ */
+ return true;
+ }
+ default:
+ throw new RuntimeException("Unknown fast decimal check precision and scale status " + status);
+ }
+ }
+
+ protected FastCheckPrecisionScaleStatus fastCheckPrecisionScale(
+ int maxPrecision, int maxScale) {
+ return
+ FastHiveDecimalImpl.fastCheckPrecisionScale(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ maxPrecision, maxScale);
+ }
+
+ protected static enum FastCheckPrecisionScaleStatus {
+ NO_CHANGE,
+ OVERFLOW,
+ UPDATE_SCALE_DOWN;
+ }
+
+ protected boolean fastUpdatePrecisionScale(
+ int maxPrecision, int maxScale, FastCheckPrecisionScaleStatus status,
+ FastHiveDecimal fastResult) {
+ return
+ FastHiveDecimalImpl.fastUpdatePrecisionScale(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ maxPrecision, maxScale, status,
+ fastResult);
+ }
+
+ protected boolean fastAdd(
+ FastHiveDecimal fastRight,
+ FastHiveDecimal fastResult) {
+ return
+ FastHiveDecimalImpl.fastAdd(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2,
+ fastRight.fastIntegerDigitCount, fastRight.fastScale,
+ fastResult);
+ }
+
+ protected boolean fastSubtract(
+ FastHiveDecimal fastRight,
+ FastHiveDecimal fastResult) {
+ return
+ FastHiveDecimalImpl.fastSubtract(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2,
+ fastRight.fastIntegerDigitCount, fastRight.fastScale,
+ fastResult);
+ }
+
+ protected boolean fastMultiply(
+ FastHiveDecimal fastRight,
+ FastHiveDecimal fastResult) {
+ return
+ FastHiveDecimalImpl.fastMultiply(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2,
+ fastRight.fastIntegerDigitCount, fastRight.fastScale,
+ fastResult);
+ }
+
+ protected boolean fastRemainder(
+ FastHiveDecimal fastRight,
+ FastHiveDecimal fastResult) {
+ return
+ FastHiveDecimalImpl.fastRemainder(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2,
+ fastRight.fastIntegerDigitCount, fastRight.fastScale,
+ fastResult);
+ }
+
+ protected boolean fastDivide(
+ FastHiveDecimal fastRight,
+ FastHiveDecimal fastResult) {
+ return
+ FastHiveDecimalImpl.fastDivide(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2,
+ fastRight.fastIntegerDigitCount, fastRight.fastScale,
+ fastResult);
+ }
+
+ protected boolean fastPow(
+ int exponent,
+ FastHiveDecimal fastResult) {
+ return
+ FastHiveDecimalImpl.fastPow(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale,
+ exponent,
+ fastResult);
+ }
+
+ protected String fastToString(
+ byte[] scratchBuffer) {
+ return
+ FastHiveDecimalImpl.fastToString(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale, -1,
+ scratchBuffer);
+ }
+
+ protected String fastToString() {
+ return
+ FastHiveDecimalImpl.fastToString(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale, -1);
+ }
+
+ protected String fastToFormatString(int formatScale) {
+ return
+ FastHiveDecimalImpl.fastToFormatString(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ formatScale);
+ }
+
+ protected String fastToFormatString(
+ int formatScale,
+ byte[] scratchBuffer) {
+ return
+ FastHiveDecimalImpl.fastToFormatString(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ formatScale,
+ scratchBuffer);
+ }
+
+ protected String fastToDigitsOnlyString() {
+ return
+ FastHiveDecimalImpl.fastToDigitsOnlyString(
+ fast0, fast1, fast2,
+ fastIntegerDigitCount);
+ }
+
+ // Sign, zero, dot, 2 * digits (to support toFormatString which can add a lot of trailing zeroes).
+ protected final static int FAST_SCRATCH_BUFFER_LEN_TO_BYTES =
+ 1 + 1 + 1 + 2 * FastHiveDecimalImpl.MAX_DECIMAL_DIGITS;
+
+ protected int fastToBytes(
+ byte[] scratchBuffer) {
+ return
+ FastHiveDecimalImpl.fastToBytes(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale, -1,
+ scratchBuffer);
+ }
+
+ protected int fastToFormatBytes(
+ int formatScale,
+ byte[] scratchBuffer) {
+ return
+ FastHiveDecimalImpl.fastToFormatBytes(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ formatScale,
+ scratchBuffer);
+ }
+
+ protected int fastToDigitsOnlyBytes(
+ byte[] scratchBuffer) {
+ return
+ FastHiveDecimalImpl.fastToDigitsOnlyBytes(
+ fast0, fast1, fast2,
+ fastIntegerDigitCount,
+ scratchBuffer);
+ }
+
+ @Override
+ public String toString() {
+ return
+ FastHiveDecimalImpl.fastToString(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale, -1);
+ }
+
+ protected boolean fastIsValid() {
+ return FastHiveDecimalImpl.fastIsValid(this);
+ }
+
+ protected void fastRaiseInvalidException() {
+ FastHiveDecimalImpl.fastRaiseInvalidException(this);
+ }
+
+ protected void fastRaiseInvalidException(String parameters) {
+ FastHiveDecimalImpl.fastRaiseInvalidException(this, parameters);
+ }
+}
\ No newline at end of file
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..84ff1c64f5c46adf8a82377ce3e6215c1370135c
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java
@@ -0,0 +1,9466 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.type;
+
+import java.util.Arrays;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.RoundingMode;
+
+/**
+ * This class is a companion to the FastHiveDecimal class that separates the essential of code
+ * out of FastHiveDecimal into static methods in this class so that they can be used directly
+ * by vectorization to implement decimals by storing the fast0, fast1, and fast2 longs and
+ * the fastSignum, fastScale, etc ints in the DecimalColumnVector class.
+ */
+public class FastHiveDecimalImpl extends FastHiveDecimal {
+
+ /**
+ * Representation of fast decimals.
+ *
+ * We use 3 long words to store the 38 digits of fast decimals and and 3 integers for sign,
+ * integer digit count, and scale.
+ *
+ * The lower and middle long words store 16 decimal digits each; the high long word has
+ * 6 decimal digits; total 38 decimal digits.
+ *
+ * We do not try and represent fast decimal value as an unsigned 128 bit binary number in 2 longs.
+ * There are several important reasons for this.
+ *
+ * The effort to represent an unsigned 128 integer in 2 Java signed longs is very difficult,
+ * error prone, hard to debug, and not worth the effort.
+ *
+ * The focus here is on reusing memory (i.e. with HiveDecimalWritable) as often as possible.
+ * Reusing memory is good for grouping of fast decimal objects and related objects in CPU cache
+ * lines for fast memory access and eliminating the cost of allocating temporary objects and
+ * reducing the global cost of garbage collection.
+ *
+ * In other words, we are focused on avoiding the poor performance of Java general immutable
+ * objects.
+ *
+ * Reducing memory size or being concerned about the memory size of using 3 longs vs. 2 longs
+ * for 128 unsigned bits is not the focus here.
+ *
+ * Besides focusing on reusing memory, storing a limited number (16) decimal digits in the longs
+ * rather than compacting the value into all binary bits of 2 longs has a surprising benefit.
+ *
+ * One big part of implementing decimals turns out to be manipulating decimal digits.
+ *
+ * For example, rounding a decimal involves trimming off lower digits or clearing lower digits.
+ * Since radix 10 digits cannot be masked with binary masks, we use division and multiplication
+ * using powers of 10. We can easily manipulate the decimal digits in a long word using simple
+ * integer multiplication / division without doing emulated 128 binary bit multiplication /
+ * division (e.g. the defunct Decimal128 class).
+ *
+ * For example, say we want to scale (round) down the fraction digits of a decimal.
+ *
+ * final long divideFactor = powerOfTenTable[scaleDown];
+ * final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown];
+ *
+ * result0 =
+ * fast0 / divideFactor
+ * + ((fast1 % divideFactor) * multiplyFactor);
+ * result1 =
+ * fast1 / divideFactor
+ * + ((fast2 % divideFactor) * multiplyFactor);
+ * result2 =
+ * fast2 / divideFactor;
+ *
+ * It also turns out to do addition and subtraction of decimals with different scales can involve
+ * overlap using more than 3 long words. Manipulating extra words is a natural extension of
+ * the existing techniques.
+ *
+ * Why is the decimal digits representation easier to debug? You can see the decimal digits in
+ * the 3 long words and do not have to convert binary words to decimal to see the value.
+ *
+ * 16 decimal digits for a long was choose so that an int can have 1/2 or 8 decimal digits during
+ * multiplication of int half words so intermediate multiplication results do not overflow a long.
+ * And, so addition overflow is well below the sign bit of a long.
+ */
+
+ // Code Sections:
+ // Initialize (fastSetFrom*).
+ // Take Integer or Fractional Portion.
+ // Binary to Decimal Conversion.
+ // Decimal to Binary Conversion.r
+ // Emulate SerializationUtils Deserialization used by ORC.
+ // Emulate SerializationUtils Serialization used by ORC.
+ // Emulate BigInteger Deserialization used by LazyBinary and others.
+ // Emulate BigInteger Serialization used by LazyBinary and others.
+ // Decimal to Integer Conversion.
+ // Decimal to Non-Integer Conversion.
+ // Decimal Comparison.
+ // Decimal Rounding.
+ // Decimal Scale Up/Down.
+ // Decimal Precision / Trailing Zeroes.
+ // Decimal Addition / Subtraction.
+ // Decimal Multiply.
+ // Decimal Division / Remainder.
+ // Decimal String Formatting.
+ // Decimal Validation.
+ // Decimal Debugging.
+
+ private static final long[] powerOfTenTable = {
+ 1L, // 0
+ 10L,
+ 100L,
+ 1000L,
+ 10000L,
+ 100000L,
+ 1000000L,
+ 10000000L,
+ 100000000L, // 8
+ 1000000000L,
+ 10000000000L,
+ 100000000000L,
+ 1000000000000L,
+ 10000000000000L,
+ 100000000000000L,
+ 1000000000000000L,
+ 10000000000000000L, // 16
+ 100000000000000000L,
+ 1000000000000000000L, // 18
+ };
+
+ public static final int MAX_DECIMAL_DIGITS = 38;
+
+ /**
+ * Int: 8 decimal digits. An even number and 1/2 of MAX_LONGWORD_DECIMAL.
+ */
+ private static final int INTWORD_DECIMAL_DIGITS = 8;
+ private static final int MULTIPLER_INTWORD_DECIMAL = (int) powerOfTenTable[INTWORD_DECIMAL_DIGITS];
+
+ /**
+ * Long: 16 decimal digits. An even number and twice MAX_INTWORD_DECIMAL.
+ */
+ private static final int LONGWORD_DECIMAL_DIGITS = 16;
+ private static final long MAX_LONGWORD_DECIMAL = powerOfTenTable[LONGWORD_DECIMAL_DIGITS] - 1;
+ private static final long MULTIPLER_LONGWORD_DECIMAL = powerOfTenTable[LONGWORD_DECIMAL_DIGITS];
+
+ public static final int DECIMAL64_DECIMAL_DIGITS = 18;
+ public static final long MAX_ABS_DECIMAL64 = 999999999999999999L; // 18 9's -- quite reliable!
+
+ private static final int TWO_X_LONGWORD_DECIMAL_DIGITS = 2 * LONGWORD_DECIMAL_DIGITS;
+ private static final int THREE_X_LONGWORD_DECIMAL_DIGITS = 3 * LONGWORD_DECIMAL_DIGITS;
+ private static final int FOUR_X_LONGWORD_DECIMAL_DIGITS = 4 * LONGWORD_DECIMAL_DIGITS;
+
+ // 38 decimal maximum - 32 digits in 2 lower longs (6 digits here).
+ private static final int HIGHWORD_DECIMAL_DIGITS = MAX_DECIMAL_DIGITS - TWO_X_LONGWORD_DECIMAL_DIGITS;
+ private static final long MAX_HIGHWORD_DECIMAL =
+ powerOfTenTable[HIGHWORD_DECIMAL_DIGITS] - 1;
+
+ // 38 * 2 or 76 full decimal maximum - (64 + 8) digits in 4 lower longs (4 digits here).
+ private static final long FULL_MAX_HIGHWORD_DECIMAL =
+ powerOfTenTable[MAX_DECIMAL_DIGITS * 2 - (FOUR_X_LONGWORD_DECIMAL_DIGITS + INTWORD_DECIMAL_DIGITS)] - 1;
+
+ /**
+ * BigInteger constants.
+ */
+
+ private static final BigInteger BIG_INTEGER_TWO = BigInteger.valueOf(2);
+ private static final BigInteger BIG_INTEGER_FIVE = BigInteger.valueOf(5);
+ private static final BigInteger BIG_INTEGER_TEN = BigInteger.valueOf(10);
+
+ public static final BigInteger BIG_INTEGER_MAX_DECIMAL =
+ BIG_INTEGER_TEN.pow(MAX_DECIMAL_DIGITS).subtract(BigInteger.ONE);
+
+ private static final BigInteger BIG_INTEGER_MAX_LONGWORD_DECIMAL =
+ BigInteger.valueOf(MAX_LONGWORD_DECIMAL);
+
+ private static final BigInteger BIG_INTEGER_LONGWORD_MULTIPLIER =
+ BigInteger.ONE.add(BIG_INTEGER_MAX_LONGWORD_DECIMAL);
+ private static final BigInteger BIG_INTEGER_LONGWORD_MULTIPLIER_2X =
+ BIG_INTEGER_LONGWORD_MULTIPLIER.multiply(BIG_INTEGER_LONGWORD_MULTIPLIER);
+ private static final BigInteger BIG_INTEGER_MAX_HIGHWORD_DECIMAL =
+ BigInteger.valueOf(MAX_HIGHWORD_DECIMAL);
+ private static final BigInteger BIG_INTEGER_HIGHWORD_MULTIPLIER =
+ BigInteger.ONE.add(BIG_INTEGER_MAX_HIGHWORD_DECIMAL);
+
+ // UTF-8 byte constants used by string/UTF-8 bytes to decimal and decimal to String/UTF-8 byte
+ // conversion.
+
+ // There is only one blank in UTF-8.
+ private static final byte BYTE_BLANK = (byte) ' ';
+
+ private static final byte BYTE_DIGIT_ZERO = (byte) '0';
+ private static final byte BYTE_DIGIT_NINE = (byte) '9';
+
+ // Decimal point.
+ private static final byte BYTE_DOT = (byte) '.';
+
+ // Sign.
+ private static final byte BYTE_MINUS = (byte) '-';
+ private static final byte BYTE_PLUS = (byte) '+';
+
+ // Exponent E or e.
+ private static final byte BYTE_EXPONENT_LOWER = (byte) 'e';
+ private static final byte BYTE_EXPONENT_UPPER = (byte) 'E';
+
+ //************************************************************************************************
+ // Initialize (fastSetFrom*).
+
+ /*
+ * All of the fastSetFrom* methods require the caller to pass a fastResult parameter has been
+ * reset for better performance.
+ */
+
+ private static void doRaiseSetFromBytesInvalid(
+ byte[] bytes, int offset, int length,
+ FastHiveDecimal fastResult) {
+ final int end = offset + length;
+ throw new RuntimeException(
+ "Invalid fast decimal \"" +
+ new String(bytes, offset, end) + "\"" +
+ " fastSignum " + fastResult.fastSignum + " fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2 +
+ " fastIntegerDigitCount " + fastResult.fastIntegerDigitCount +" fastScale " + fastResult.fastScale +
+ " stack trace: " + getStackTraceAsSingleLine(Thread.currentThread().getStackTrace()));
+ }
+
+ /**
+ * Scan a byte array slice for a decimal number in UTF-8 bytes.
+ *
+ * Syntax:
+ * [+|-][integerPortion][.[fractionalDigits]][{E|e}[+|-]exponent]
+ * // Where at least one integer or fractional
+ * // digit is required...
+ *
+ * We handle too many fractional digits by doing rounding ROUND_HALF_UP.
+ *
+ * NOTE: The fastSetFromBytes method requires the caller to pass a fastResult parameter has been
+ * reset for better performance.
+ *
+ * @param bytes the bytes to copy from
+ * @param offset the starting location in bytes
+ * @param length the number of bytes to use from bytes
+ * @param trimBlanks should spaces be trimmed?
+ * @param fastResult True if the byte array slice was successfully converted to a decimal.
+ * @return Was a valid number found?
+ */
+ public static boolean fastSetFromBytes(byte[] bytes, int offset, int length, boolean trimBlanks,
+ FastHiveDecimal fastResult) {
+
+ final int bytesLength = bytes.length;
+
+ if (offset < 0 || offset >= bytesLength) {
+ return false;
+ }
+ final int end = offset + length;
+ if (end <= offset || end > bytesLength) {
+ return false;
+ }
+
+ // We start here with at least one byte.
+ int index = offset;
+
+ if (trimBlanks) {
+ while (bytes[index] == BYTE_BLANK) {
+ if (++index >= end) {
+ return false;
+ }
+ }
+ }
+
+ // Started with a few ideas from BigDecimal(char[] in, int offset, int len) constructor...
+ // But soon became very fast decimal specific.
+
+ boolean isNegative = false;
+ if (bytes[index] == BYTE_MINUS) {
+ isNegative = true;
+ if (++index >= end) {
+ return false;
+ }
+ } else if (bytes[index] == BYTE_PLUS) {
+ if (++index >= end) {
+ return false;
+ }
+ }
+
+ int precision = 0;
+
+ // We fill starting with highest digit in highest longword (HIGHWORD_DECIMAL_DIGITS) and
+ // move down. At end will will shift everything down if necessary.
+
+ int longWordIndex = 0; // Where 0 is the highest longword; 1 is middle longword, etc.
+
+ int digitNum = HIGHWORD_DECIMAL_DIGITS;
+ long multiplier = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1];
+
+ int digitValue = 0;
+ long longWord = 0;
+
+ long fast0 = 0;
+ long fast1 = 0;
+ long fast2 = 0;
+
+ byte work;
+
+ // Parse integer portion.
+
+ boolean haveInteger = false;
+ while (true) {
+ work = bytes[index];
+ if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) {
+ break;
+ }
+ haveInteger = true;
+ if (precision == 0 && work == BYTE_DIGIT_ZERO) {
+ // Ignore leading zeroes.
+ if (++index >= end) {
+ break;
+ }
+ continue;
+ }
+ digitValue = work - BYTE_DIGIT_ZERO;
+ if (digitNum == 0) {
+
+ // Integer parsing move to next lower longword.
+
+ // Save previous longword.
+ if (longWordIndex == 0) {
+ fast2 = longWord;
+ } else if (longWordIndex == 1) {
+ fast1 = longWord;
+ } else if (longWordIndex == 2) {
+
+ // We have filled HiveDecimal.MAX_PRECISION digits and have no more room in our limit precision
+ // fast decimal.
+ return false;
+ }
+ longWordIndex++;
+
+ // The middle and lowest longwords highest digit number is LONGWORD_DECIMAL_DIGITS.
+ digitNum = LONGWORD_DECIMAL_DIGITS;
+ multiplier = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - 1];
+ longWord = 0;
+ }
+ longWord += digitValue * multiplier;
+ multiplier /= 10;
+ digitNum--;
+ precision++;
+ if (++index >= end) {
+ break;
+ }
+ }
+
+ // At this point we may have parsed an integer.
+
+ // Try to eat a dot now since it could be the end. We remember if we saw a dot so we can
+ // do error checking later and detect just a dot.
+ boolean sawDot = false;
+ if (index < end && bytes[index] == BYTE_DOT) {
+ sawDot = true;
+ index++;
+ }
+
+ // Try to eat trailing blank padding.
+ if (trimBlanks && index < end && bytes[index] == BYTE_BLANK) {
+ index++;
+ while (index < end && bytes[index] == BYTE_BLANK) {
+ index++;
+ }
+ if (index < end) {
+ // Junk after trailing blank padding.
+ return false;
+ }
+ // Otherwise, fall through and process the what we saw before possible trailing blanks.
+ }
+
+ // Any more input?
+ if (index >= end) {
+
+ // We hit the end after getting optional integer and optional dot and optional blank padding.
+
+ if (!haveInteger) {
+ return false;
+ }
+
+ if (precision == 0) {
+
+ // We just had leading zeroes (and possibly a dot and trailing blanks).
+ // Value is 0.
+ return true;
+ }
+ // Save last longword.
+ if (longWordIndex == 0) {
+ fast2 = longWord;
+ } else if (longWordIndex == 1) {
+ fast1 = longWord;
+ } else {
+ fast0 = longWord;
+ }
+ fastResult.fastSignum = (isNegative ? -1 : 1);
+ fastResult.fastIntegerDigitCount = precision;
+ fastResult.fastScale = 0;
+ final int scaleDown = HiveDecimal.MAX_PRECISION - precision;
+ if (scaleDown > 0) {
+ doFastScaleDown(fast0, fast1, fast2, scaleDown, fastResult);
+ } else {
+ fastResult.fast0 = fast0;
+ fastResult.fast1 = fast1;
+ fastResult.fast2 = fast2;
+ }
+ return true;
+ }
+
+ // We have more input but did we start with something valid?
+ if (!haveInteger && !sawDot) {
+
+ // Must have one of those at this point.
+ return false;
+ }
+
+ int integerDigitCount = precision;
+
+ int nonTrailingZeroScale = 0;
+ boolean roundingNecessary = false;
+ if (sawDot) {
+
+ // Parse fraction portion.
+
+ while (true) {
+ work = bytes[index];
+ if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) {
+ if (!haveInteger) {
+
+ // Naked dot.
+ return false;
+ }
+ break;
+ }
+ digitValue = work - BYTE_DIGIT_ZERO;
+ if (digitNum == 0) {
+
+ // Fraction digit parsing move to next lower longword.
+
+ // Save previous longword.
+ if (longWordIndex == 0) {
+ fast2 = longWord;
+ } else if (longWordIndex == 1) {
+ fast1 = longWord;
+ } else if (longWordIndex == 2) {
+
+ // We have filled HiveDecimal.MAX_PRECISION digits and have no more room in our limit precision
+ // fast decimal. However, since we are processing fractional digits, we do rounding.
+ // away.
+ if (digitValue >= 5) {
+ roundingNecessary = true;
+ }
+
+ // Scan through any remaining digits...
+ while (++index < end) {
+ work = bytes[index];
+ if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) {
+ break;
+ }
+ }
+ break;
+ }
+ longWordIndex++;
+ digitNum = LONGWORD_DECIMAL_DIGITS;
+ multiplier = powerOfTenTable[digitNum - 1];
+ longWord = 0;
+ }
+ longWord += digitValue * multiplier;
+ multiplier /= 10;
+ digitNum--;
+ precision++;
+ if (digitValue != 0) {
+ nonTrailingZeroScale = precision - integerDigitCount;
+ }
+ if (++index >= end) {
+ break;
+ }
+ }
+ }
+
+ boolean haveExponent = false;
+ if (index < end &&
+ (bytes[index] == BYTE_EXPONENT_UPPER || bytes[index] == BYTE_EXPONENT_LOWER)) {
+ haveExponent = true;
+ index++;
+ if (index >= end) {
+ // More required.
+ return false;
+ }
+ }
+
+ // At this point we have a number. Save it in fastResult. Round it. If we have an exponent,
+ // we will do a power 10 operation on fastResult.
+
+ // Save last longword.
+ if (longWordIndex == 0) {
+ fast2 = longWord;
+ } else if (longWordIndex == 1) {
+ fast1 = longWord;
+ } else {
+ fast0 = longWord;
+ }
+
+ int trailingZeroesScale = precision - integerDigitCount;
+ if (integerDigitCount == 0 && nonTrailingZeroScale == 0) {
+ // Zero(es).
+ } else {
+ fastResult.fastSignum = (isNegative ? -1 : 1);
+ fastResult.fastIntegerDigitCount = integerDigitCount;
+ fastResult.fastScale = nonTrailingZeroScale;
+ final int trailingZeroCount = trailingZeroesScale - fastResult.fastScale;
+ final int scaleDown = HiveDecimal.MAX_PRECISION - precision + trailingZeroCount;
+ if (scaleDown > 0) {
+ doFastScaleDown(fast0, fast1, fast2, scaleDown, fastResult);
+ } else {
+ fastResult.fast0 = fast0;
+ fastResult.fast1 = fast1;
+ fastResult.fast2 = fast2;
+ }
+ }
+
+ if (roundingNecessary) {
+
+ if (fastResult.fastSignum == 0) {
+ fastResult.fastSignum = (isNegative ? -1 : 1);
+ fastResult.fast0 = 1;
+ fastResult.fastIntegerDigitCount = 0;
+ fastResult.fastScale = HiveDecimal.MAX_SCALE;
+ } else {
+ if (!fastAdd(
+ fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ fastResult.fastIntegerDigitCount, fastResult.fastScale,
+ fastResult.fastSignum, 1, 0, 0, 0, trailingZeroesScale,
+ fastResult)) {
+ return false;
+ }
+ }
+ }
+
+ if (!haveExponent) {
+
+ // Try to eat trailing blank padding.
+ if (trimBlanks && index < end && bytes[index] == BYTE_BLANK) {
+ index++;
+ while (index < end && bytes[index] == BYTE_BLANK) {
+ index++;
+ }
+ }
+ if (index < end) {
+ // Junk after trailing blank padding.
+ return false;
+ }
+ return true;
+ }
+
+ // At this point, we have seen the exponent letter E or e and have decimal information as:
+ // isNegative, precision, integerDigitCount, nonTrailingZeroScale, and
+ // fast0, fast1, fast2.
+ //
+ // After we determine the exponent, we will do appropriate scaling and fill in fastResult.
+
+ boolean isExponentNegative = false;
+ if (bytes[index] == BYTE_MINUS) {
+ isExponentNegative = true;
+ if (++index >= end) {
+ return false;
+ }
+ } else if (bytes[index] == BYTE_PLUS) {
+ if (++index >= end) {
+ return false;
+ }
+ }
+
+ long exponent = 0;
+ multiplier = 1;
+ while (true) {
+ work = bytes[index];
+ if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) {
+ break;
+ }
+ if (multiplier > 10) {
+ // Power of ten way beyond our precision/scale...
+ return false;
+ }
+ digitValue = work - BYTE_DIGIT_ZERO;
+ if (digitValue != 0 || exponent != 0) {
+ exponent = exponent * 10 + digitValue;
+ multiplier *= 10;
+ }
+ if (++index >= end) {
+ break;
+ }
+ }
+ if (isExponentNegative) {
+ exponent = -exponent;
+ }
+
+ // Try to eat trailing blank padding.
+ if (trimBlanks && index < end && bytes[index] == BYTE_BLANK) {
+ index++;
+ while (index < end && bytes[index] == BYTE_BLANK) {
+ index++;
+ }
+ }
+ if (index < end) {
+ // Junk after exponent.
+ return false;
+ }
+
+
+ if (integerDigitCount == 0 && nonTrailingZeroScale == 0) {
+ // Zero(es).
+ return true;
+ }
+
+ if (exponent == 0) {
+
+ // No effect since 10^0 = 1.
+
+ } else {
+
+ // We for these input with exponents, we have at this point an intermediate decimal,
+ // an exponent power, and a result:
+ //
+ // intermediate
+ // input decimal exponent result
+ // 701E+1 701 scale 0 +1 7010 scale 0
+ // 3E+4 3 scale 0 +4 3 scale 0
+ // 3.223E+9 3.223 scale 3 +9 3223000000 scale 0
+ // 0.009E+10 0.009 scale 4 +10 90000000 scale 0
+ // 0.3221E-2 0.3221 scale 4 -2 0.003221 scale 6
+ // 0.00223E-20 0.00223 scale 5 -20 0.0000000000000000000000223 scale 25
+ //
+
+ if (!fastScaleByPowerOfTen(
+ fastResult,
+ (int) exponent,
+ fastResult)) {
+ return false;
+ }
+ }
+
+ final int trailingZeroCount =
+ fastTrailingDecimalZeroCount(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ fastResult.fastIntegerDigitCount, fastResult.fastScale);
+ if (trailingZeroCount > 0) {
+ doFastScaleDown(
+ fastResult,
+ trailingZeroCount,
+ fastResult);
+ fastResult.fastScale -= trailingZeroCount;
+ }
+
+ return true;
+ }
+
+ /**
+ * Scans a byte array slice for UNSIGNED RAW DIGITS ONLY in UTF-8 (ASCII) characters
+ * and forms a decimal from the digits and a sign and scale.
+ *
+ * Designed for BinarySortable serialization format that separates the sign and scale
+ * from the raw digits.
+ *
+ * NOTE: The fastSetFromDigitsOnlyBytesAndScale method requires the caller to pass a fastResult
+ * parameter has been reset for better performance.
+ *
+ * @param isNegative is the number negative
+ * @param bytes the bytes to read from
+ * @param offset the position to start at
+ * @param length the number of bytes to read
+ * @param scale the scale of the number
+ * @param fastResult an object it into
+ * @return True if the sign, digits, and scale were successfully converted to a decimal.
+ */
+ public static boolean fastSetFromDigitsOnlyBytesAndScale(
+ boolean isNegative, byte[] bytes, int offset, int length, int scale,
+ FastHiveDecimal fastResult) {
+
+ final int bytesLength = bytes.length;
+
+ if (offset < 0 || offset >= bytesLength) {
+ return false;
+ }
+ final int end = offset + length;
+ if (end <= offset || end > bytesLength) {
+ return false;
+ }
+
+ // We start here with at least one byte.
+ int index = offset;
+
+ // A stripped down version of fastSetFromBytes.
+
+ int precision = 0;
+
+ // We fill starting with highest digit in highest longword (HIGHWORD_DECIMAL_DIGITS) and
+ // move down. At end will will shift everything down if necessary.
+
+ int longWordIndex = 0; // Where 0 is the highest longword; 1 is middle longword, etc.
+
+ int digitNum = HIGHWORD_DECIMAL_DIGITS;
+ long multiplier = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1];
+
+ int digitValue;
+ long longWord = 0;
+
+ long fast0 = 0;
+ long fast1 = 0;
+ long fast2 = 0;
+
+ byte work;
+
+ // Parse digits.
+
+ boolean haveInteger = false;
+ while (true) {
+ work = bytes[index];
+ if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) {
+ if (!haveInteger) {
+ return false;
+ }
+ break;
+ }
+ haveInteger = true;
+ if (precision == 0 && work == BYTE_DIGIT_ZERO) {
+ // Ignore leading zeroes.
+ if (++index >= end) {
+ break;
+ }
+ continue;
+ }
+ digitValue = work - BYTE_DIGIT_ZERO;
+ if (digitNum == 0) {
+
+ // Integer parsing move to next lower longword.
+
+ // Save previous longword.
+ if (longWordIndex == 0) {
+ fast2 = longWord;
+ } else if (longWordIndex == 1) {
+ fast1 = longWord;
+ } else if (longWordIndex == 2) {
+
+ // We have filled HiveDecimal.MAX_PRECISION digits and have no more room in our limit precision
+ // fast decimal.
+ return false;
+ }
+ longWordIndex++;
+
+ // The middle and lowest longwords highest digit number is LONGWORD_DECIMAL_DIGITS.
+ digitNum = LONGWORD_DECIMAL_DIGITS;
+ multiplier = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - 1];
+ longWord = 0;
+ }
+ longWord += digitValue * multiplier;
+ multiplier /= 10;
+ digitNum--;
+ precision++;
+ if (++index >= end) {
+ break;
+ }
+ }
+
+ // Just an digits?
+ if (index < end) {
+ return false;
+ }
+
+ if (precision == 0) {
+ // We just had leading zeroes.
+ // Value is 0.
+ return true;
+ }
+
+ // Save last longword.
+ if (longWordIndex == 0) {
+ fast2 = longWord;
+ } else if (longWordIndex == 1) {
+ fast1 = longWord;
+ } else {
+ fast0 = longWord;
+ }
+ fastResult.fastSignum = (isNegative ? -1 : 1);
+ fastResult.fastIntegerDigitCount = Math.max(0, precision - scale);
+ fastResult.fastScale = scale;
+ final int scaleDown = HiveDecimal.MAX_PRECISION - precision;
+ if (scaleDown > 0) {
+ doFastScaleDown(fast0, fast1, fast2, scaleDown, fastResult);
+ } else {
+ fastResult.fast0 = fast0;
+ fastResult.fast1 = fast1;
+ fastResult.fast2 = fast2;
+ }
+ return true;
+
+ }
+
+ /**
+ * Scale down a BigInteger by a power of 10 and round off if necessary using ROUND_HALF_UP.
+ * @return The scaled and rounded BigInteger.
+ */
+ private static BigInteger doBigIntegerScaleDown(BigInteger unscaledValue, int scaleDown) {
+ BigInteger[] quotientAndRemainder = unscaledValue.divideAndRemainder(BigInteger.TEN.pow(scaleDown));
+ BigInteger quotient = quotientAndRemainder[0];
+ BigInteger round = quotientAndRemainder[1].divide(BigInteger.TEN.pow(scaleDown - 1));
+ if (round.compareTo(BIG_INTEGER_FIVE) >= 0) {
+ quotient = quotient.add(BigInteger.ONE);
+ }
+ return quotient;
+ }
+
+ /**
+ * Create a fast decimal from a BigDecimal.
+ *
+ * NOTE: The fastSetFromBigDecimal method requires the caller to pass a fastResult
+ * parameter has been reset for better performance.
+ *
+ * @param bigDecimal the big decimal to copy
+ * @param allowRounding is rounding allowed?
+ * @param fastResult an object to reuse
+ * @return True if the BigDecimal could be converted to our decimal representation.
+ */
+ public static boolean fastSetFromBigDecimal(
+ BigDecimal bigDecimal, boolean allowRounding, FastHiveDecimal fastResult) {
+
+ // We trim the trailing zero fraction digits so we don't cause unnecessary precision
+ // overflow later.
+ if (bigDecimal.signum() == 0) {
+ if (bigDecimal.scale() != 0) {
+
+ // For some strange reason BigDecimal 0 can have a scale. We do not support that.
+ bigDecimal = BigDecimal.ZERO;
+ }
+ }
+
+ if (!allowRounding) {
+ if (bigDecimal.signum() != 0) {
+ BigDecimal bigDecimalStripped = bigDecimal.stripTrailingZeros();
+ int stripTrailingZerosScale = bigDecimalStripped.scale();
+ // System.out.println("FAST_SET_FROM_BIG_DECIMAL bigDecimal " + bigDecimal);
+ // System.out.println("FAST_SET_FROM_BIG_DECIMAL bigDecimalStripped " + bigDecimalStripped);
+ // System.out.println("FAST_SET_FROM_BIG_DECIMAL stripTrailingZerosScale " + stripTrailingZerosScale);
+ if (stripTrailingZerosScale < 0) {
+
+ // The trailing zeroes extend into the integer part -- we only want to eliminate the
+ // fractional zero digits.
+
+ bigDecimal = bigDecimal.setScale(0);
+ } else {
+
+ // Ok, use result with some or all fractional digits stripped.
+
+ bigDecimal = bigDecimalStripped;
+ }
+ }
+ int scale = bigDecimal.scale();
+ if (scale < 0 || scale > HiveDecimal.MAX_SCALE) {
+ return false;
+ }
+ // The digits must fit without rounding.
+ if (!fastSetFromBigInteger(bigDecimal.unscaledValue(), fastResult)) {
+ return false;
+ }
+ if (fastResult.fastSignum != 0) {
+ fastResult.fastIntegerDigitCount = Math.max(0, fastResult.fastIntegerDigitCount - scale);
+ fastResult.fastScale = scale;
+ }
+ return true;
+ }
+ // This method will scale down and round to fit, if necessary.
+ if (!fastSetFromBigInteger(bigDecimal.unscaledValue(), bigDecimal.scale(),
+ bigDecimal.precision(), fastResult)) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Scan a String for a decimal number in UTF-8 characters.
+ *
+ * NOTE: The fastSetFromString method requires the caller to pass a fastResult
+ * parameter has been reset for better performance.
+ *
+ * @param string the string to parse
+ * @param trimBlanks should the blanks be trimmed
+ * @param result an object to reuse
+ * @return True if the String was successfully converted to a decimal.
+ */
+ public static boolean fastSetFromString(
+ String string, boolean trimBlanks, FastHiveDecimal result) {
+ byte[] bytes = string.getBytes();
+ return fastSetFromBytes(bytes, 0, bytes.length, trimBlanks, result);
+ }
+
+ /**
+ * Creates a scale 0 fast decimal from an int.
+ *
+ * NOTE: The fastSetFromString method requires the caller to pass a fastResult
+ * parameter has been reset for better performance.
+ * @param intValue the value to set
+ * @param fastResult an object to reuse
+ */
+ public static void fastSetFromInt(int intValue, FastHiveDecimal fastResult) {
+ if (intValue == 0) {
+ // Zero special case.
+ return;
+ }
+ if (intValue > 0) {
+ fastResult.fastSignum = 1;
+ } else {
+ fastResult.fastSignum = -1;
+ intValue = Math.abs(intValue);
+ }
+ // 10 digit int is all in lowest 16 decimal digit longword.
+ // Since we are creating with scale 0, no fraction digits to zero trim.
+ fastResult.fast0 = intValue & 0xFFFFFFFFL;
+ fastResult.fastIntegerDigitCount =
+ fastLongWordPrecision(fastResult.fast0);
+ }
+
+ /**
+ * Creates a scale 0 fast decimal from a long.
+ *
+ * NOTE: The fastSetFromLong method requires the caller to pass a fastResult
+ * parameter has been reset for better performance.
+ *
+ * @param longValue the value to set
+ * @param fastResult an object to reuse
+ */
+ public static void fastSetFromLong(
+ long longValue, FastHiveDecimal fastResult) {
+ if (longValue == 0) {
+ // Zero special case.
+ return;
+ }
+ // Handle minimum integer case that doesn't have abs().
+ if (longValue == Long.MIN_VALUE) {
+ // Split -9,223,372,036,854,775,808 into 16 digit middle and lowest longwords by hand.
+ fastResult.fastSignum = -1;
+ fastResult.fast1 = 922L;
+ fastResult.fast0 = 3372036854775808L;
+ fastResult.fastIntegerDigitCount = 19;
+ } else {
+ if (longValue > 0) {
+ fastResult.fastSignum = 1;
+ } else {
+ fastResult.fastSignum = -1;
+ longValue = Math.abs(longValue);
+ }
+ // Split into 16 digit middle and lowest longwords remainder / division.
+ fastResult.fast1 = longValue / MULTIPLER_LONGWORD_DECIMAL;
+ fastResult.fast0 = longValue % MULTIPLER_LONGWORD_DECIMAL;
+ if (fastResult.fast1 != 0) {
+ fastResult.fastIntegerDigitCount =
+ LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast1);
+ } else {
+ fastResult.fastIntegerDigitCount =
+ fastLongWordPrecision(fastResult.fast0);
+ }
+ }
+ return;
+ }
+
+ /**
+ * Creates a fast decimal from a long with a specified scale.
+ *
+ * NOTE: The fastSetFromLongAndScale method requires the caller to pass a fastResult
+ * parameter has been reset for better performance.
+ *
+ * @param longValue the value to set as a long
+ * @param scale the scale to use
+ * @param fastResult an object to reuse
+ * @return was the conversion successful?
+ */
+ public static boolean fastSetFromLongAndScale(
+ long longValue, int scale, FastHiveDecimal fastResult) {
+
+ if (scale < 0 || scale > HiveDecimal.MAX_SCALE) {
+ return false;
+ }
+
+ fastSetFromLong(longValue, fastResult);
+ if (scale == 0) {
+ return true;
+ }
+
+ if (!fastScaleByPowerOfTen(
+ fastResult,
+ -scale,
+ fastResult)) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Creates fast decimal from a float.
+ *
+ * NOTE: The fastSetFromFloat method requires the caller to pass a fastResult
+ * parameter has been reset for better performance.
+ *
+ * @param floatValue the value to set
+ * @param fastResult an object to reuse
+ * @return was the conversion successful?
+ */
+ public static boolean fastSetFromFloat(
+ float floatValue, FastHiveDecimal fastResult) {
+
+ String floatString = Float.toString(floatValue);
+ return fastSetFromString(floatString, false, fastResult);
+
+ }
+
+ /**
+ * Creates fast decimal from a double.
+ *
+ * NOTE: The fastSetFromDouble method requires the caller to pass a fastResult
+ * parameter has been reset for better performance.
+ *
+ * @param doubleValue the value to set
+ * @param fastResult an object to reuse
+ * @return was the conversion successful?
+ */
+ public static boolean fastSetFromDouble(
+ double doubleValue, FastHiveDecimal fastResult) {
+
+ String doubleString = Double.toString(doubleValue);
+ return fastSetFromString(doubleString, false, fastResult);
+
+ }
+
+ /**
+ * Creates a fast decimal from a BigInteger with scale 0.
+ *
+ * For efficiency, we assume that fastResult is fastReset. This method does not set the
+ * fastScale field.
+ *
+ * NOTE: The fastSetFromBigInteger method requires the caller to pass a fastResult
+ * parameter has been reset for better performance.
+ *
+ * @param bigInteger the value to set
+ * @param fastResult an object to reuse
+ * @return Return true if the BigInteger value fit within HiveDecimal.MAX_PRECISION. Otherwise,
+ * false for overflow.
+ */
+ public static boolean fastSetFromBigInteger(
+ BigInteger bigInteger, FastHiveDecimal fastResult) {
+
+ final int signum = bigInteger.signum();
+ if (signum == 0) {
+ // Zero special case.
+ return true;
+ }
+ fastResult.fastSignum = signum;
+ if (signum == -1) {
+ bigInteger = bigInteger.negate();
+ }
+ if (bigInteger.compareTo(BIG_INTEGER_LONGWORD_MULTIPLIER) < 0) {
+
+ // Fits in one longword.
+ fastResult.fast0 = bigInteger.longValue();
+ if (fastResult.fast0 == 0) {
+ fastResult.fastSignum = 0;
+ } else {
+ fastResult.fastIntegerDigitCount = fastLongWordPrecision(fastResult.fast0);
+ }
+ return true;
+ }
+ BigInteger[] quotientAndRemainder =
+ bigInteger.divideAndRemainder(BIG_INTEGER_LONGWORD_MULTIPLIER);
+ fastResult.fast0 = quotientAndRemainder[1].longValue();
+ BigInteger quotient = quotientAndRemainder[0];
+ if (quotient.compareTo(BIG_INTEGER_LONGWORD_MULTIPLIER) < 0) {
+
+ // Fits in two longwords.
+ fastResult.fast1 = quotient.longValue();
+ if (fastResult.fast0 == 0 && fastResult.fast1 == 0) {
+ // The special case zero logic at the beginning should have caught this.
+ throw new RuntimeException("Unexpected");
+ } else {
+ fastResult.fastIntegerDigitCount =
+ LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast1);
+ }
+ return true;
+ }
+
+ // Uses all 3 decimal longs.
+ quotientAndRemainder =
+ quotient.divideAndRemainder(BIG_INTEGER_LONGWORD_MULTIPLIER);
+ fastResult.fast1 = quotientAndRemainder[1].longValue();
+ quotient = quotientAndRemainder[0];
+ if (quotient.compareTo(BIG_INTEGER_HIGHWORD_MULTIPLIER) >= 0) {
+ // Overflow.
+ return false;
+ }
+ fastResult.fast2 = quotient.longValue();
+ if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) {
+ fastResult.fastSignum = 0;
+ } else {
+ fastResult.fastIntegerDigitCount =
+ TWO_X_LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(fastResult.fast2);
+ }
+ return true;
+ }
+
+ /**
+ * Creates a fast decimal from a BigInteger with a specified scale.
+ *
+ * NOTE: The fastSetFromBigInteger method requires the caller to pass a fastResult
+ * parameter has been reset for better performance.
+ *
+ * @param bigInteger the value to set as an integer
+ * @param scale the scale to use
+ * @param fastResult an object to reuse
+ * @return True if the BigInteger and scale were successfully converted to a decimal.
+ */
+ public static boolean fastSetFromBigInteger(
+ BigInteger bigInteger, int scale, FastHiveDecimal fastResult) {
+ // Poor performance, because the precision will be calculated by bigInteger.toString()
+ return fastSetFromBigInteger(bigInteger, scale, -1, fastResult);
+ }
+
+ /**
+ * Creates a fast decimal from a BigInteger with a specified scale.
+ *
+ * NOTE: The fastSetFromBigInteger method requires the caller to pass a fastResult
+ * parameter has been reset for better performance.
+ *
+ * @param bigInteger the value to set as an integer
+ * @param scale the scale to use
+ * @param precision the precision to use
+ * @param fastResult an object to reuse
+ * @return True if the BigInteger and scale were successfully converted to a decimal.
+ */
+ public static boolean fastSetFromBigInteger(
+ BigInteger bigInteger, int scale, int precision, FastHiveDecimal fastResult) {
+
+ if (scale < 0) {
+
+ // Multiply by 10^(-scale) to normalize. We do not use negative scale in our representation.
+ //
+ // Example:
+ // 4.172529E+20 has a negative scale -20 since scale is number of digits below the dot.
+ // 417252900000000000000 normalized as scale 0.
+ //
+ bigInteger = bigInteger.multiply(BIG_INTEGER_TEN.pow(-scale));
+ scale = 0;
+ }
+
+ int signum = bigInteger.signum();
+ if (signum == 0) {
+ // Zero.
+ return true;
+ } else if (signum == -1) {
+ // Normalize to positive.
+ bigInteger = bigInteger.negate();
+ }
+
+ if (precision < 0) {
+ // A slow way to get the number of decimal digits.
+ precision = bigInteger.toString().length();
+ }
+
+ // System.out.println("FAST_SET_FROM_BIG_INTEGER adjusted bigInteger " + bigInteger + " precision " + precision);
+
+ int integerDigitCount = precision - scale;
+ // System.out.println("FAST_SET_FROM_BIG_INTEGER integerDigitCount " + integerDigitCount + " scale " + scale);
+ int maxScale;
+ if (integerDigitCount >= 0) {
+ if (integerDigitCount > HiveDecimal.MAX_PRECISION) {
+ return false;
+ }
+ maxScale = HiveDecimal.MAX_SCALE - integerDigitCount;
+ } else {
+ maxScale = HiveDecimal.MAX_SCALE;
+ }
+ // System.out.println("FAST_SET_FROM_BIG_INTEGER maxScale " + maxScale);
+
+ if (scale > maxScale) {
+
+ // A larger scale is ok -- we will knock off lower digits and round.
+
+ final int trimAwayCount = scale - maxScale;
+ // System.out.println("FAST_SET_FROM_BIG_INTEGER trimAwayCount " + trimAwayCount);
+ if (trimAwayCount > 1) {
+ // First, throw away digits below round digit.
+ BigInteger bigIntegerThrowAwayBelowRoundDigitDivisor = BIG_INTEGER_TEN.pow(trimAwayCount - 1);
+ bigInteger = bigInteger.divide(bigIntegerThrowAwayBelowRoundDigitDivisor);
+ }
+ // System.out.println("FAST_SET_FROM_BIG_INTEGER with round digit bigInteger " + bigInteger + " length " + bigInteger.toString().length());
+
+ BigInteger[] quotientAndRemainder = bigInteger.divideAndRemainder(BIG_INTEGER_TEN);
+ // System.out.println("FAST_SET_FROM_BIG_INTEGER quotientAndRemainder " + Arrays.toString(quotientAndRemainder));
+
+ BigInteger quotient = quotientAndRemainder[0];
+ if (quotientAndRemainder[1].intValue() >= 5) {
+ if (quotient.equals(BIG_INTEGER_MAX_DECIMAL)) {
+
+ // 38 9's digits.
+ // System.out.println("FAST_SET_FROM_BIG_INTEGER quotient is BIG_INTEGER_MAX_DECIMAL");
+
+ if (maxScale == 0) {
+ // No room above for rounding.
+ return false;
+ }
+
+ // System.out.println("FAST_SET_FROM_BIG_INTEGER reached here... scale " + scale + " maxScale " + maxScale);
+ // Rounding results in 10^N.
+ bigInteger = BIG_INTEGER_TEN.pow(integerDigitCount);
+ maxScale = 0;
+ } else {
+
+ // Round up.
+ bigInteger = quotient.add(BigInteger.ONE);
+ }
+ } else {
+
+ // No rounding.
+ bigInteger = quotient;
+ }
+ scale = maxScale;
+ }
+ if (!fastSetFromBigInteger(bigInteger, fastResult)) {
+ return false;
+ }
+
+ if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) {
+ fastResult.fastSignum = 0;
+ } else {
+ fastResult.fastSignum = signum;
+ fastResult.fastIntegerDigitCount = Math.max(0, fastResult.fastIntegerDigitCount - scale);
+ fastResult.fastScale = scale;
+
+ final int trailingZeroCount =
+ fastTrailingDecimalZeroCount(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ fastResult.fastIntegerDigitCount, scale);
+ if (trailingZeroCount > 0) {
+ doFastScaleDown(
+ fastResult,
+ trailingZeroCount,
+ fastResult);
+ fastResult.fastScale -= trailingZeroCount;
+ }
+ }
+
+ return true;
+ }
+
+ //************************************************************************************************
+ // Take Integer or Fractional Portion.
+
+ /**
+ * Creates fast decimal from the fraction portion of a fast decimal.
+ *
+ * NOTE: The fastFractionPortion method requires the caller to pass a fastResult
+ * parameter has been reset for better performance.
+ *
+ * @param fastSignum the sign of the number (1, 0, or -1)
+ * @param fast0 high bits
+ * @param fast1 second word bits
+ * @param fast2 third word bits
+ * @param fastScale the scale
+ * @param fastResult an object to reuse
+ */
+ public static void fastFractionPortion(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastScale,
+ FastHiveDecimal fastResult) {
+
+ if (fastSignum == 0 || fastScale == 0) {
+ fastResult.fastReset();
+ return;
+ }
+
+ // Clear integer portion; keep fraction.
+
+ // Adjust all longs using power 10 division/remainder.
+ long result0;
+ long result1;
+ long result2;
+ if (fastScale < LONGWORD_DECIMAL_DIGITS) {
+
+ // Part of lowest word survives.
+
+ final long clearFactor = powerOfTenTable[fastScale];
+
+ result0 = fast0 % clearFactor;
+ result1 = 0;
+ result2 = 0;
+
+ } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ // Throw away lowest word.
+
+ final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS;
+
+ final long clearFactor = powerOfTenTable[adjustedScaleDown];
+
+ result0 = fast0;
+ result1 = fast1 % clearFactor;
+ result2 = 0;
+
+ } else {
+
+ // Throw away middle and lowest words.
+
+ final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS;
+
+ final long clearFactor = powerOfTenTable[adjustedScaleDown];
+
+ result0 = fast0;
+ result1 = fast1;
+ result2 = fast2 % clearFactor;
+
+ }
+ if (result0 == 0 && result1 == 0 && result2 == 0) {
+ fastResult.fastReset();
+ } else {
+ fastResult.fastSet(fastSignum, result0, result1, result2, /* fastIntegerDigitCount */ 0, fastScale);
+ }
+ }
+
+ /**
+ * Creates fast decimal from the integer portion.
+ *
+ * NOTE: The fastFractionPortion method requires the caller to pass a fastResult
+ * parameter has been reset for better performance.
+ *
+ * @param fastSignum the sign of the number (1, 0, or -1)
+ * @param fast0 high bits
+ * @param fast1 second word bits
+ * @param fast2 third word bits
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale
+ * @param fastResult an object to reuse
+ */
+ public static void fastIntegerPortion(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ FastHiveDecimal fastResult) {
+
+ if (fastSignum == 0) {
+ fastResult.fastReset();
+ return;
+ }
+ if (fastScale == 0) {
+ fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ }
+
+ // Scale down no rounding to clear fraction.
+ fastResult.fastSignum = fastSignum;
+ doFastScaleDown(
+ fast0, fast1, fast2,
+ fastScale,
+ fastResult);
+ fastResult.fastIntegerDigitCount = fastIntegerDigitCount;
+ fastResult.fastScale = 0;
+ }
+
+ //************************************************************************************************
+ // Binary to Decimal Conversion.
+
+ /**
+ * Convert 3 binary words of N bits each to a fast decimal (scale 0).
+ *
+ * The 3 binary words highWord, middleWord, and lowerWord form a large binary value:
+ *
+ * highWord * 2^(M+L) + middleWord * 2^L + lowerWord.
+ *
+ * Where L is the number of bits in the lower word; M is the number of bits in the middle word.
+ * We let L and M be different to support the SerializationUtil serialization where the lower
+ * word is 62 bits and the remaining words are 63 bits...
+ *
+ * @param lowerWord the lower internal representation
+ * @param middleWord the middle internal representation
+ * @param highWord the high internal representation
+ * @param middleWordMultiplier 2^L
+ * @param highWordMultiplier 2^(M+L)
+ * @param fastResult an object to reuse
+ * @return True if the conversion of the 3 binary words to decimal was successful.
+ */
+ public static boolean doBinaryToDecimalConversion(
+ long lowerWord, long middleWord, long highWord,
+ FastHiveDecimal middleWordMultiplier,
+ FastHiveDecimal highWordMultiplier,
+ FastHiveDecimal fastResult) {
+
+ /*
+ * Challenge: How to do the math to get this raw binary back to our decimal form.
+ *
+ * Briefly, for the middle and upper binary words, convert the middle/upper word into a decimal
+ * long words and then multiply those by the binary word's power of 2.
+ *
+ * And, add the multiply results into the result decimal longwords.
+ *
+ */
+ long result0 =
+ lowerWord % MULTIPLER_LONGWORD_DECIMAL;
+ long result1 =
+ lowerWord / MULTIPLER_LONGWORD_DECIMAL;
+ long result2 = 0;
+
+ if (middleWord != 0 || highWord != 0) {
+
+ if (highWord == 0) {
+
+ // Form result from lower and middle words.
+
+ if (!fastMultiply5x5HalfWords(
+ middleWord % MULTIPLER_LONGWORD_DECIMAL,
+ middleWord / MULTIPLER_LONGWORD_DECIMAL,
+ 0,
+ middleWordMultiplier.fast0, middleWordMultiplier.fast1, middleWordMultiplier.fast2,
+ fastResult)) {
+ return false;
+ }
+
+ final long calc0 =
+ result0
+ + fastResult.fast0;
+ result0 =
+ calc0 % MULTIPLER_LONGWORD_DECIMAL;
+ final long calc1 =
+ calc0 / MULTIPLER_LONGWORD_DECIMAL
+ + result1
+ + fastResult.fast1;
+ result1 =
+ calc1 % MULTIPLER_LONGWORD_DECIMAL;
+ result2 =
+ calc1 / MULTIPLER_LONGWORD_DECIMAL
+ + fastResult.fast2;
+
+ } else if (middleWord == 0) {
+
+ // Form result from lower and high words.
+
+ if (!fastMultiply5x5HalfWords(
+ highWord % MULTIPLER_LONGWORD_DECIMAL,
+ highWord / MULTIPLER_LONGWORD_DECIMAL,
+ 0,
+ highWordMultiplier.fast0, highWordMultiplier.fast1, highWordMultiplier.fast2,
+ fastResult)) {
+ return false;
+ }
+
+ final long calc0 =
+ result0
+ + fastResult.fast0;
+ result0 =
+ calc0 % MULTIPLER_LONGWORD_DECIMAL;
+ final long calc1 =
+ calc0 / MULTIPLER_LONGWORD_DECIMAL
+ + result1
+ + fastResult.fast1;
+ result1 =
+ calc1 % MULTIPLER_LONGWORD_DECIMAL;
+ result2 =
+ calc1 / MULTIPLER_LONGWORD_DECIMAL
+ + fastResult.fast2;
+
+ } else {
+
+ // Form result from lower, middle, and middle words.
+
+ if (!fastMultiply5x5HalfWords(
+ middleWord % MULTIPLER_LONGWORD_DECIMAL,
+ middleWord / MULTIPLER_LONGWORD_DECIMAL,
+ 0,
+ middleWordMultiplier.fast0, middleWordMultiplier.fast1, middleWordMultiplier.fast2,
+ fastResult)) {
+ return false;
+ }
+
+ long middleResult0 = fastResult.fast0;
+ long middleResult1 = fastResult.fast1;
+ long middleResult2 = fastResult.fast2;
+
+ if (!fastMultiply5x5HalfWords(
+ highWord % MULTIPLER_LONGWORD_DECIMAL,
+ highWord / MULTIPLER_LONGWORD_DECIMAL,
+ 0,
+ highWordMultiplier.fast0, highWordMultiplier.fast1, highWordMultiplier.fast2,
+ fastResult)) {
+ return false;
+ }
+
+ long calc0 =
+ result0
+ + middleResult0
+ + fastResult.fast0;
+ result0 =
+ calc0 % MULTIPLER_LONGWORD_DECIMAL;
+ long calc1 =
+ calc0 / MULTIPLER_LONGWORD_DECIMAL
+ + result1
+ + middleResult1
+ + fastResult.fast1;
+ result1 =
+ calc1 % MULTIPLER_LONGWORD_DECIMAL;
+ result2 =
+ calc1 / MULTIPLER_LONGWORD_DECIMAL
+ + middleResult2
+ + fastResult.fast2;
+ }
+ }
+
+ // Let caller set negative sign if necessary.
+ if (result2 != 0) {
+ fastResult.fastIntegerDigitCount = TWO_X_LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(result2);
+ fastResult.fastSignum = 1;
+ } else if (result1 != 0) {
+ fastResult.fastIntegerDigitCount = LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(result1);
+ fastResult.fastSignum = 1;
+ } else if (result0 != 0) {
+ fastResult.fastIntegerDigitCount = fastHighWordPrecision(result0);
+ fastResult.fastSignum = 1;
+ } else {
+ fastResult.fastIntegerDigitCount = 0;
+ fastResult.fastSignum = 0;
+ }
+
+ fastResult.fast0 = result0;
+ fastResult.fast1 = result1;
+ fastResult.fast2 = result2;
+
+ return true;
+ }
+
+ //************************************************************************************************
+ // Decimal to Binary Conversion.
+
+ /**
+ * A helper method that produces a single binary word remainder from a fast decimal (and
+ * quotient).
+ *
+ * The fast decimal is longwords of 16 digits each and we need binary words of 2^N. Since
+ * we are in decimal form, we have do work to get to convert to binary form.
+ *
+ * We effectively need to produce on big binary value (i.e. greater than 64 bits since
+ * HiveDecimal needs 128 bits of binary which Java does not provide primitive support for)
+ * from the decimal long words and get the lower N binary bit remainder.
+ *
+ * We could try and do decimal division by 2^N to get the (integer) quotient, multiply the
+ * quotient by 2^N decimal, and finally do a decimal subtract that from the original decimal.
+ * The resulting decimal can be used to easily get the binary remainder.
+ *
+ * However, currently, we do not have fast decimal division.
+ *
+ * The "trick" we do here is to remember from your Algebra in school than multiplication and
+ * division are inverses of each other.
+ *
+ * So instead of doing decimal division by 2^N we multiply by the inverse: 2^-N.
+ *
+ * We produce 1 binary word (remainder) and a decimal quotient for the higher portion.
+ *
+ * @param dividendFast0 The input decimal that will produce a
+ * single binary word remainder and decimal quotient.
+ * @param dividendFast1 second word
+ * @param dividendFast2 third word
+ * @param fastInverseConst the fast decimal inverse of 2^N = 2^-N
+ * @param quotientIntegerWordNum the word in the inverse multiplication result
+ * to find the quotient integer decimal portion
+ * @param quotientIntegerDigitNum the digit in the result to find the quotient
+ * integer decimal portion
+ * @param fastMultiplierConst The fast decimal multiplier for converting the
+ * quotient integer to the larger number to
+ * subtract from the input decimal to get the
+ * remainder.
+ * @param scratchLongs where to store the result remainder word (index 3) and
+ * result quotient decimal longwords (indices 0 .. 2)
+ * @return True if the results were produced without overflow.
+ */
+ public static boolean doDecimalToBinaryDivisionRemainder(
+ long dividendFast0, long dividendFast1, long dividendFast2,
+ FastHiveDecimal fastInverseConst,
+ int quotientIntegerWordNum,
+ int quotientIntegerDigitNum,
+ FastHiveDecimal fastMultiplierConst,
+ long[] scratchLongs) {
+
+ // Multiply by inverse (2^-N) to do the 2^N division.
+ if (!fastMultiply5x6HalfWords(
+ dividendFast0, dividendFast1, dividendFast2,
+ fastInverseConst.fast0, fastInverseConst.fast1, fastInverseConst.fast2,
+ scratchLongs)) {
+ // Overflow.
+ return false;
+ }
+
+ final long divideFactor = powerOfTenTable[quotientIntegerDigitNum];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - quotientIntegerDigitNum];
+
+ // Extract the integer portion to get the quotient.
+ long quotientFast0 =
+ scratchLongs[quotientIntegerWordNum] / divideFactor
+ + ((scratchLongs[quotientIntegerWordNum + 1] % divideFactor) * multiplyFactor);
+ long quotientFast1 =
+ scratchLongs[quotientIntegerWordNum + 1] / divideFactor
+ + ((scratchLongs[quotientIntegerWordNum + 2] % divideFactor) * multiplyFactor);
+ long quotientFast2 =
+ scratchLongs[quotientIntegerWordNum + 2] / divideFactor;
+
+ // Multiply the integer quotient back out so we can subtract it from the original to get
+ // the remainder.
+ if (!fastMultiply5x6HalfWords(
+ quotientFast0, quotientFast1, quotientFast2,
+ fastMultiplierConst.fast0, fastMultiplierConst.fast1, fastMultiplierConst.fast2,
+ scratchLongs)) {
+ return false;
+ }
+
+ long quotientMultiplied0 = scratchLongs[0];
+ long quotientMultiplied1 = scratchLongs[1];
+ long quotientMultiplied2 = scratchLongs[2];
+
+ if (!doSubtractSameScaleNoUnderflow(
+ dividendFast0, dividendFast1, dividendFast2,
+ quotientMultiplied0, quotientMultiplied1, quotientMultiplied2,
+ scratchLongs)) {
+ // Underflow.
+ return false;
+ }
+
+ long remainderBinaryWord =
+ scratchLongs[1] * MULTIPLER_LONGWORD_DECIMAL
+ + scratchLongs[0];
+
+ // Pack the output into the scratch longs.
+ scratchLongs[0] = quotientFast0;
+ scratchLongs[1] = quotientFast1;
+ scratchLongs[2] = quotientFast2;
+
+ scratchLongs[3] = remainderBinaryWord;
+
+ return true;
+ }
+
+ /**
+ * Convert a fast decimal into 3 binary words of N bits each.
+ *
+ * The 3 binary words will form a large binary value that is the unsigned unscaled decimal value:
+ *
+ * highWord * 2^(M+L) + middleWord * 2^L + lowerWord.
+ *
+ * Where L is the number of bits in the lower word; M is the number of bits in the middle word.
+ * We let L and M be different to support the SerializationUtil serialization where the lower
+ * word is 62 bits and the remaining words are 63 bits...
+ *
+ * The fast decimal is longwords of 16 digits each and we need binary words of 2^N. Since
+ * we are in decimal form, we have do work to get to convert to binary form.
+ *
+ * See the comments for doDecimalToBinaryDivisionRemainder for details on the parameters.
+ *
+ * The lowerWord is produced by calling doDecimalToBinaryDivisionRemainder. The quotient from
+ * that is passed to doDecimalToBinaryDivisionRemainder to produce the middleWord. The final
+ * quotient is used to produce the highWord.
+ *
+ * @return True if the 3 binary words were produced without overflow. Overflow is not expected.
+ */
+ private static boolean doDecimalToBinaryConversion(
+ long fast0, long fast1, long fast2,
+ FastHiveDecimal fastInverseConst,
+ int quotientIntegerWordNum,
+ int quotientIntegerDigitNum,
+ FastHiveDecimal fastMultiplierConst,
+ long[] scratchLongs) {
+
+ long lowerBinaryWord;
+ long middleBinaryWord = 0;
+ long highBinaryWord = 0;
+
+ if (fastCompareTo(
+ 1,
+ fast0, fast1, fast2, 0,
+ 1,
+ fastMultiplierConst.fast0, fastMultiplierConst.fast1, fastMultiplierConst.fast2, 0) < 0) {
+
+ // Optimize: whole decimal fits in one binary word.
+
+ lowerBinaryWord =
+ fast1 * MULTIPLER_LONGWORD_DECIMAL
+ + fast0;
+
+ } else {
+
+ // Do division/remainder to get lower binary word; quotient will either be middle decimal
+ // or be both high and middle decimal that requires another division/remainder.
+
+ if (!doDecimalToBinaryDivisionRemainder(
+ fast0, fast1, fast2,
+ fastInverseConst,
+ quotientIntegerWordNum,
+ quotientIntegerDigitNum,
+ fastMultiplierConst,
+ scratchLongs)) {
+ // Overflow.
+ return false;
+ }
+
+ // Unpack the output.
+ long quotientFast0 = scratchLongs[0];
+ long quotientFast1 = scratchLongs[1];
+ long quotientFast2 = scratchLongs[2];
+
+ lowerBinaryWord = scratchLongs[3];
+
+ if (fastCompareTo(
+ 1,
+ quotientFast0, quotientFast1, quotientFast2, 0,
+ 1,
+ fastMultiplierConst.fast0, fastMultiplierConst.fast1, fastMultiplierConst.fast2, 0) < 0) {
+
+ // Optimize: whole decimal fits in two binary words.
+
+ middleBinaryWord =
+ quotientFast1 * MULTIPLER_LONGWORD_DECIMAL
+ + quotientFast0;
+
+ } else {
+ if (!doDecimalToBinaryDivisionRemainder(
+ quotientFast0, quotientFast1, quotientFast2,
+ fastInverseConst,
+ quotientIntegerWordNum,
+ quotientIntegerDigitNum,
+ fastMultiplierConst,
+ scratchLongs)) {
+ // Overflow.
+ return false;
+ }
+
+ highBinaryWord =
+ scratchLongs[1] * MULTIPLER_LONGWORD_DECIMAL
+ + scratchLongs[0];
+
+ middleBinaryWord = scratchLongs[3];
+
+ }
+ }
+
+ scratchLongs[0] = lowerBinaryWord;
+ scratchLongs[1] = middleBinaryWord;
+ scratchLongs[2] = highBinaryWord;
+
+ return true;
+ }
+
+ //************************************************************************************************
+ // Emulate SerializationUtils Deserialization used by ORC.
+
+ /*
+ * fastSerializationUtilsRead lower word is 62 bits (the lower bit is used as the sign and is
+ * removed). So, we need a multiplier 2^62
+ *
+ * 2^62 =
+ * 4611686018427387904 or
+ * 4,611,686,018,427,387,904 or
+ * 461,1686018427387904 (16 digit comma'd)
+ */
+ private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_62 =
+ new FastHiveDecimal(1, 1686018427387904L, 461L, 0, 19, 0);
+
+ /*
+ * fastSerializationUtilsRead middle word is 63 bits. So, we need a multiplier 2^63
+ *
+ * 2^63 =
+ * 9223372036854775808 (Long.MAX_VALUE) or
+ * 9,223,372,036,854,775,808 or
+ * 922,3372036854775808 (16 digit comma'd)
+ */
+ private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_63 =
+ new FastHiveDecimal(1, 3372036854775808L, 922L, 0, 19, 0);
+
+ /*
+ * fastSerializationUtilsRead high word multiplier:
+ *
+ * Multiply by 2^(62 + 63) -- 38 digits or 3 decimal words.
+ *
+ * (2^62)*(2^63) =
+ * 42535295865117307932921825928971026432 or
+ * (12345678901234567890123456789012345678)
+ * ( 1 2 3 )
+ * 42,535,295,865,117,307,932,921,825,928,971,026,432 or
+ * 425352,9586511730793292,1825928971026432 (16 digit comma'd)
+ */
+ private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_125 =
+ new FastHiveDecimal(1, 1825928971026432L, 9586511730793292L, 425352L, 38, 0);
+
+ /*
+ * Inverse of 2^63 = 2^-63. Please see comments for doDecimalToBinaryDivisionRemainder.
+ *
+ * Multiply by 1/2^63 = 1.08420217248550443400745280086994171142578125e-19 to divide by 2^63.
+ * As 16 digit comma'd 1084202172485,5044340074528008,6994171142578125
+ *
+ * Scale down: 63 = 44 fraction digits + 19 (negative exponent or number of zeros after dot).
+ *
+ * 3*16 (48) + 15 --> 63 down shift.
+ */
+ private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_63_INVERSE =
+ new FastHiveDecimal(1, 6994171142578125L, 5044340074528008L, 1084202172485L, 45, 0);
+
+ /*
+ * Where in the inverse multiplication result to find the quotient integer decimal portion.
+ *
+ * Please see comments for doDecimalToBinaryDivisionRemainder.
+ */
+ private static final int SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_WORD_NUM = 3;
+ private static final int SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_DIGIT_NUM = 15;
+
+ /**
+ * Deserialize data written in the format used by the SerializationUtils methods
+ * readBigInteger/writeBigInteger and create a decimal using the supplied scale.
+ *
+ * ORC uses those SerializationUtils methods for its serialization.
+ *
+ * A scratch bytes array is necessary to do the binary to decimal conversion for better
+ * performance. Pass a FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ byte array for
+ * scratchBytes.
+ *
+ * @param inputStream the stream to read from
+ * @param scale the scale of the number
+ * @param scratchBytes An array for the binary to decimal conversion for better
+ * performance. Must have length of
+ * FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ.
+ * @param fastResult an object to reuse
+ * @return The deserialized decimal or null if the conversion failed.
+ * @throws IOException failures in reading the stream
+ */
+ public static boolean fastSerializationUtilsRead(InputStream inputStream, int scale,
+ byte[] scratchBytes,
+ FastHiveDecimal fastResult) throws IOException {
+
+ // Following a suggestion from Gopal, quickly read in the bytes from the stream.
+ // CONSIDER: Have ORC read the whole input stream into a big byte array with one call to
+ // the read(byte[] b, int off, int len) method and then let this method read from the big
+ // byte array.
+ int readCount = 0;
+ int input;
+ do {
+ input = inputStream.read();
+ if (input == -1) {
+ throw new EOFException("Reading BigInteger past EOF from " + inputStream);
+ }
+ scratchBytes[readCount++] = (byte) input;
+ } while (input >= 0x80);
+
+ /*
+ * Determine the 3 binary words like what SerializationUtils.readBigInteger does.
+ */
+
+ long lowerWord63 = 0;
+ long middleWord63 = 0;
+ long highWord63 = 0;
+
+ long work = 0;
+ int offset = 0;
+ int readIndex = 0;
+ long b;
+ do {
+ b = scratchBytes[readIndex++];
+ work |= (0x7f & b) << (offset % 63);
+ offset += 7;
+ // if we've read 63 bits, roll them into the result
+ if (offset == 63) {
+ lowerWord63 = work;
+ work = 0;
+ } else if (offset % 63 == 0) {
+ if (offset == 126) {
+ middleWord63 = work;
+ } else if (offset == 189) {
+ highWord63 = work;
+ } else {
+ throw new EOFException("Reading more than 3 words of BigInteger");
+ }
+ work = 0;
+ }
+ } while (readIndex < readCount);
+
+ if (work != 0) {
+ if (offset < 63) {
+ lowerWord63 = work;
+ } else if (offset < 126) {
+ middleWord63 = work;
+ } else if (offset < 189) {
+ highWord63 =work;
+ } else {
+ throw new EOFException("Reading more than 3 words of BigInteger");
+ }
+ }
+
+ // Grab sign bit and shift it away.
+ boolean isNegative = ((lowerWord63 & 0x1) != 0);
+ lowerWord63 >>= 1;
+
+ /*
+ * Use common binary to decimal conversion method we share with fastSetFromBigIntegerBytes.
+ */
+ if (!doBinaryToDecimalConversion(
+ lowerWord63, middleWord63, highWord63,
+ FAST_HIVE_DECIMAL_TWO_POWER_62,
+ FAST_HIVE_DECIMAL_TWO_POWER_125, // 2^(62 + 63)
+ fastResult)) {
+ return false;
+ }
+
+ if (isNegative) {
+
+ // Adjust negative result, again doing what SerializationUtils.readBigInteger does.
+ if (!doAddSameScaleSameSign(
+ /* resultSignum */ 1,
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ 1, 0, 0,
+ fastResult)) {
+ return false;
+ }
+ }
+
+ if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) {
+ fastResult.fastSignum = 0;
+ } else {
+ fastResult.fastSignum = (isNegative ? -1 : 1);
+ final int rawPrecision = fastRawPrecision(fastResult);
+ fastResult.fastIntegerDigitCount = Math.max(0, rawPrecision - scale);
+ fastResult.fastScale = scale;
+
+ /*
+ * Just in case we deserialize a decimal with trailing zeroes...
+ */
+ final int resultTrailingZeroCount =
+ fastTrailingDecimalZeroCount(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ fastResult.fastIntegerDigitCount, fastResult.fastScale);
+ if (resultTrailingZeroCount > 0) {
+ doFastScaleDown(
+ fastResult,
+ resultTrailingZeroCount,
+ fastResult);
+
+ fastResult.fastScale -= resultTrailingZeroCount;
+ }
+ }
+
+ return true;
+ }
+
+ //************************************************************************************************
+ // Emulate SerializationUtils Serialization used by ORC.
+
+ /**
+ * Write the value of this decimal just like SerializationUtils.writeBigInteger. It header
+ * comments are:
+ *
+ * Write the arbitrarily sized signed BigInteger in vint format.
+ *
+ * Signed integers are encoded using the low bit as the sign bit using zigzag
+ * encoding.
+ *
+ * Each byte uses the low 7 bits for data and the high bit for stop/continue.
+ *
+ * Bytes are stored LSB first.
+ *
+ * NOTE:
+ * SerializationUtils.writeBigInteger sometimes pads the result with extra zeroes due to
+ * BigInteger.bitLength -- we do not emulate that. SerializationUtils.readBigInteger will
+ * produce the same result for both.
+ *
+ * @param outputStream the stream to write to
+ * @param fastSignum the sign digit (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1 of the internal representation
+ * @param fast2 word 2 of the internal representation
+ * @param fastIntegerDigitCount unused
+ * @param fastScale unused
+ * @param scratchLongs scratch space
+ * @return True if the decimal was successfully serialized into the output stream.
+ * @throws IOException for problems in writing
+ */
+ public static boolean fastSerializationUtilsWrite(OutputStream outputStream,
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ long[] scratchLongs)
+ throws IOException {
+
+ boolean isNegative = (fastSignum == -1);
+
+ /*
+ * The sign is encoded as the least significant bit.
+ *
+ * We need to adjust our decimal before conversion to binary.
+ *
+ * Positive:
+ * Multiply by 2.
+ *
+ * Negative:
+ * Logic in SerializationUtils.writeBigInteger does a negate on the BigInteger. We
+ * do not have to since FastHiveDecimal stores the numbers unsigned in fast0, fast1,
+ * and fast2. We do need to subtract one though.
+ *
+ * And then multiply by 2 and add in the 1 sign bit.
+ *
+ * CONSIDER: This could be combined.
+ */
+ long adjust0;
+ long adjust1;
+ long adjust2;
+
+ if (isNegative) {
+
+ // Subtract 1.
+ long r0 = fast0 - 1;
+ long r1;
+ if (r0 < 0) {
+ adjust0 = r0 + MULTIPLER_LONGWORD_DECIMAL;
+ r1 = fast1 - 1;
+ } else {
+ adjust0 = r0;
+ r1 = fast1;
+ }
+ if (r1 < 0) {
+ adjust1 = r1 + MULTIPLER_LONGWORD_DECIMAL;
+ adjust2 = fast2 - 1;
+ } else {
+ adjust1 = r1;
+ adjust2 = fast2;
+ }
+ if (adjust2 < 0) {
+ return false;
+ }
+
+ // Now multiply by 2 and add 1 sign bit.
+ r0 = adjust0 * 2 + 1;
+ adjust0 =
+ r0 % MULTIPLER_LONGWORD_DECIMAL;
+ r1 =
+ adjust1 * 2
+ + r0 / MULTIPLER_LONGWORD_DECIMAL;
+ adjust1 =
+ r1 % MULTIPLER_LONGWORD_DECIMAL;
+ adjust2 =
+ adjust2 * 2
+ + r1 / MULTIPLER_LONGWORD_DECIMAL;
+
+ } else {
+
+ // Multiply by 2 to make room for 0 sign bit.
+ long r0 = fast0 * 2;
+ adjust0 =
+ r0 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r1 =
+ fast1 * 2
+ + r0 / MULTIPLER_LONGWORD_DECIMAL;
+ adjust1 =
+ r1 % MULTIPLER_LONGWORD_DECIMAL;
+ adjust2 =
+ fast2 * 2
+ + r1 / MULTIPLER_LONGWORD_DECIMAL;
+
+ }
+
+ /*
+ * Use common decimal to binary conversion method we share with fastBigIntegerBytes.
+ */
+ if (!doDecimalToBinaryConversion(
+ adjust0, adjust1, adjust2,
+ FAST_HIVE_DECIMAL_TWO_POWER_63_INVERSE,
+ SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_WORD_NUM,
+ SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_DIGIT_NUM,
+ FAST_HIVE_DECIMAL_TWO_POWER_63,
+ scratchLongs)) {
+ // Overflow.
+ return false;
+ }
+
+ long lowerWord63 = scratchLongs[0];
+ long middleWord63 = scratchLongs[1];
+ long highWord63 = scratchLongs[2];
+
+ int wordCount;
+ if (highWord63 != 0) {
+ wordCount = 3;
+ } else if (middleWord63 != 0) {
+ wordCount = 2;
+ } else {
+ wordCount = 1;
+ }
+
+ // Write out the first 63 bits worth of data.
+ long lowBits = lowerWord63;
+ for(int i=0; i < 9; ++i) {
+ // If this is the last byte, leave the high bit off
+ if (wordCount == 1 && (lowBits & ~0x7f) == 0) {
+ outputStream.write((byte) lowBits);
+ return true;
+ } else {
+ outputStream.write((byte) (0x80 | (lowBits & 0x7f)));
+ lowBits >>>= 7;
+ }
+ }
+ if (wordCount <= 1) {
+ throw new RuntimeException("Expecting write word count > 1");
+ }
+
+ lowBits = middleWord63;
+ for(int i=0; i < 9; ++i) {
+ // If this is the last byte, leave the high bit off
+ if (wordCount == 2 && (lowBits & ~0x7f) == 0) {
+ outputStream.write((byte) lowBits);
+ return true;
+ } else {
+ outputStream.write((byte) (0x80 | (lowBits & 0x7f)));
+ lowBits >>>= 7;
+ }
+ }
+
+ lowBits = highWord63;
+ for(int i=0; i < 9; ++i) {
+ // If this is the last byte, leave the high bit off
+ if ((lowBits & ~0x7f) == 0) {
+ outputStream.write((byte) lowBits);
+ return true;
+ } else {
+ outputStream.write((byte) (0x80 | (lowBits & 0x7f)));
+ lowBits >>>= 7;
+ }
+ }
+
+ // Should not get here.
+ throw new RuntimeException("Unexpected");
+ }
+
+ public static long getDecimal64AbsMax(int precision) {
+ return powerOfTenTable[precision] - 1;
+ }
+
+ /*
+ * Deserializes 64-bit decimals up to the maximum 64-bit precision (18 decimal digits).
+ *
+ * NOTE: Major assumption: the input decimal64 has already been bounds checked and a least
+ * has a precision <= DECIMAL64_DECIMAL_DIGITS. We do not bounds check here for better
+ * performance.
+ */
+ public static void fastDeserialize64(
+ final long inputDecimal64Long, final int inputScale,
+ FastHiveDecimal fastResult) {
+
+ long decimal64Long;
+ if (inputDecimal64Long == 0) {
+ fastResult.fastReset();
+ return;
+ } else if (inputDecimal64Long > 0) {
+ fastResult.fastSignum = 1;
+ decimal64Long = inputDecimal64Long;
+ } else {
+ fastResult.fastSignum = -1;
+ decimal64Long = -inputDecimal64Long;
+ }
+
+ // Trim trailing zeroes -- but only below the decimal point.
+ int trimScale = inputScale;
+ while (trimScale > 0 && decimal64Long % 10 == 0) {
+ decimal64Long /= 10;
+ trimScale--;
+ }
+
+ fastResult.fast2 = 0;
+ fastResult.fast1 = decimal64Long / MULTIPLER_LONGWORD_DECIMAL;
+ fastResult.fast0 = decimal64Long % MULTIPLER_LONGWORD_DECIMAL;
+
+ fastResult.fastScale = trimScale;
+
+ fastResult.fastIntegerDigitCount =
+ Math.max(0, fastRawPrecision(fastResult) - fastResult.fastScale);
+ }
+
+ /*
+ * Serializes decimal64 up to the maximum 64-bit precision (18 decimal digits).
+ *
+ * NOTE: Major assumption: the fast decimal has already been bounds checked and a least
+ * has a precision <= DECIMAL64_DECIMAL_DIGITS. We do not bounds check here for better
+ * performance.
+ */
+ public static long fastSerialize64(
+ int scale,
+ int fastSignum, long fast1, long fast0, int fastScale) {
+
+ if (fastSignum == 0) {
+ return 0;
+ } else if (fastSignum == 1) {
+ return (fast1 * MULTIPLER_LONGWORD_DECIMAL + fast0) * powerOfTenTable[scale - fastScale];
+ } else {
+ return -(fast1 * MULTIPLER_LONGWORD_DECIMAL + fast0) * powerOfTenTable[scale - fastScale];
+ }
+ }
+
+ //************************************************************************************************
+ // Emulate BigInteger deserialization used by LazyBinary and others.
+
+ /*
+ * fastSetFromBigIntegerBytes word size we choose is 56 bits to stay below the 64 bit sign bit:
+ * So, we need a multiplier 2^56
+ *
+ * 2^56 =
+ * 72057594037927936 or
+ * 72,057,594,037,927,936 or
+ * 7,2057594037927936 (16 digit comma'd)
+ */
+ private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_56 =
+ new FastHiveDecimal(1, 2057594037927936L, 7L, 0, 17, 0);
+
+ /*
+ * fastSetFromBigIntegerBytes high word multiplier is 2^(56 + 56)
+ *
+ * (2^56)*(2^56) =
+ * 5192296858534827628530496329220096 or
+ * (1234567890123456789012345678901234)
+ * ( 1 2 3 )
+ * 5,192,296,858,534,827,628,530,496,329,220,096 or
+ * 51,9229685853482762,8530496329220096 (16 digit comma'd)
+ */
+ private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_112 =
+ new FastHiveDecimal(1, 8530496329220096L, 9229685853482762L, 51L, 34, 0);
+
+ // Multiply by 1/2^56 or 1.387778780781445675529539585113525390625e-17 to divide by 2^56.
+ // As 16 digit comma'd 13877787,8078144567552953,9585113525390625
+ //
+ // Scale down: 56 = 39 fraction digits + 17 (negative exponent or number of zeros after dot).
+ //
+ // 3*16 (48) + 8 --> 56 down shift.
+ //
+ private static final FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_56_INVERSE =
+ new FastHiveDecimal(1, 9585113525390625L, 8078144567552953L, 13877787L, 40, 0);
+
+ /*
+ * Where in the inverse multiplication result to find the quotient integer decimal portion.
+ *
+ * Please see comments for doDecimalToBinaryDivisionRemainder.
+ */
+ private static final int BIG_INTEGER_BYTES_QUOTIENT_INTEGER_WORD_NUM = 3;
+ private static final int BIG_INTEGER_BYTES_QUOTIENT_INTEGER_DIGIT_NUM = 8;
+
+ private static final int INITIAL_SHIFT = 48; // 56 bits minus 1 byte.
+
+ // Long masks and values.
+ private static final long LONG_56_BIT_MASK = 0xFFFFFFFFFFFFFFL;
+ private static final long LONG_TWO_TO_56_POWER = LONG_56_BIT_MASK + 1L;
+ private static final long LONG_BYTE_MASK = 0xFFL;
+ private static final long LONG_BYTE_HIGH_BIT_MASK = 0x80L;
+
+ // Byte values.
+ private static final byte BYTE_ALL_BITS = (byte) 0xFF;
+
+ /**
+ * Convert bytes in the format used by BigInteger's toByteArray format (and accepted by its
+ * constructor) into a decimal using the specified scale.
+ *
+ * Our bigIntegerBytes methods create bytes in this format, too.
+ *
+ * This method is designed for high performance and does not create an actual BigInteger during
+ * binary to decimal conversion.
+ *
+ * @param bytes the bytes to read from
+ * @param offset the starting position in the bytes array
+ * @param length the number of bytes to read from the bytes array
+ * @param scale the scale of the number
+ * @param fastResult an object to reused
+ * @return did the conversion succeed?
+ */
+ public static boolean fastSetFromBigIntegerBytesAndScale(
+ byte[] bytes, int offset, int length, int scale,
+ FastHiveDecimal fastResult) {
+
+ final int bytesLength = bytes.length;
+
+ if (offset < 0 || offset >= bytesLength) {
+ return false;
+ }
+ final int end = offset + length;
+ if (end <= offset || end > bytesLength) {
+ return false;
+ }
+
+ final int startOffset = offset;
+
+ // Roughly based on BigInteger code.
+
+ boolean isNegative = (bytes[offset] < 0);
+ if (isNegative) {
+
+ // Find first non-sign (0xff) byte of input.
+ while (offset < end) {
+ if (bytes[offset] != -1) {
+ break;
+ }
+ offset++;
+ }
+ if (offset > end) {
+ return false;
+ }
+ } else {
+
+ // Strip leading zeroes -- although there shouldn't be any for a decimal.
+
+ while (offset < end && bytes[offset] == 0) {
+ offset++;
+ }
+ if (offset >= end) {
+ // Zero.
+ return true;
+ }
+ }
+
+ long lowerWord56 = 0;
+ long middleWord56 = 0;
+ long highWord56 = 0;
+
+ int reverseIndex = end;
+
+ long work;
+ int shift;
+
+ final int lowestCount = Math.min(reverseIndex - offset, 7);
+ shift = 0;
+ for (int i = 0; i < lowestCount; i++) {
+ work = bytes[--reverseIndex] & 0xFF;
+ lowerWord56 |= work << shift;
+ shift += 8;
+ }
+
+ if (reverseIndex <= offset) {
+ if (isNegative) {
+ lowerWord56 = ~lowerWord56 & ((1L << shift) - 1);
+ }
+ } else {
+
+ // Go on to middle word.
+
+ final int middleCount = Math.min(reverseIndex - offset, 7);
+ shift = 0;
+ for (int i = 0; i < middleCount; i++) {
+ work = bytes[--reverseIndex] & 0xFF;
+ middleWord56 |= work << shift;
+ shift += 8;
+ }
+ if (reverseIndex <= offset) {
+ if (isNegative) {
+ lowerWord56 = ~lowerWord56 & LONG_56_BIT_MASK;
+ middleWord56 = ~middleWord56 & ((1L << shift) - 1);
+ }
+ } else {
+
+ // Go on to high word.
+
+ final int highCount = Math.min(reverseIndex - offset, 7);
+ shift = 0;
+ for (int i = 0; i < highCount; i++) {
+ work = bytes[--reverseIndex] & 0xFF;
+ highWord56 |= work << shift;
+ shift += 8;
+ }
+ if (isNegative) {
+ // We only need to apply negation to all 3 words when there are 3 words, etc.
+ lowerWord56 = ~lowerWord56 & LONG_56_BIT_MASK;
+ middleWord56 = ~middleWord56 & LONG_56_BIT_MASK;
+ highWord56 = ~highWord56 & ((1L << shift) - 1);
+ }
+ }
+ }
+
+ if (!doBinaryToDecimalConversion(
+ lowerWord56, middleWord56, highWord56,
+ FAST_HIVE_DECIMAL_TWO_POWER_56,
+ FAST_HIVE_DECIMAL_TWO_POWER_112, // 2^(56 + 56)
+ fastResult)) {
+ // Overflow. Use slower alternate.
+ return doAlternateSetFromBigIntegerBytesAndScale(
+ bytes, startOffset, length, scale,
+ fastResult);
+ }
+
+ // System.out.println("fastSetFromBigIntegerBytesAndScale fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2);
+ if (isNegative) {
+ if (!doAddSameScaleSameSign(
+ /* resultSignum */ 1,
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ 1, 0, 0,
+ fastResult)) {
+ // Overflow. Use slower alternate.
+ return doAlternateSetFromBigIntegerBytesAndScale(
+ bytes, startOffset, length, scale,
+ fastResult);
+ }
+ }
+
+ if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) {
+ fastResult.fastSignum = 0;
+ } else {
+ fastResult.fastSignum = (isNegative ? -1 : 1);
+ fastResult.fastScale = scale;
+ final int rawPrecision = fastRawPrecision(fastResult);
+ fastResult.fastIntegerDigitCount = Math.max(0, rawPrecision - scale);
+
+ /*
+ * Just in case we deserialize a decimal with trailing zeroes...
+ */
+ final int resultTrailingZeroCount =
+ fastTrailingDecimalZeroCount(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ fastResult.fastIntegerDigitCount, fastResult.fastScale);
+ if (resultTrailingZeroCount > 0) {
+ doFastScaleDown(
+ fastResult,
+ resultTrailingZeroCount,
+ fastResult);
+
+ fastResult.fastScale -= resultTrailingZeroCount;
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * When fastSetFromBigIntegerBytesAndScale can handle the input because it is too large,
+ * we fall back to this.
+ */
+ private static boolean doAlternateSetFromBigIntegerBytesAndScale(
+ byte[] bytes, int offset, int length, int scale,
+ FastHiveDecimal fastResult) {
+
+ byte[] byteArray = Arrays.copyOfRange(bytes, offset, offset + length);
+
+ BigInteger bigInteger = new BigInteger(byteArray);
+ // System.out.println("doAlternateSetFromBigIntegerBytesAndScale bigInteger " + bigInteger);
+ BigDecimal bigDecimal = new BigDecimal(bigInteger, scale);
+ // System.out.println("doAlternateSetFromBigIntegerBytesAndScale bigDecimal " + bigDecimal);
+ fastResult.fastReset();
+ return fastSetFromBigDecimal(bigDecimal, true, fastResult);
+ }
+
+ //************************************************************************************************
+ // Emulate BigInteger serialization used by LazyBinary, Avro, Parquet, and possibly others.
+
+ public static int fastBigIntegerBytes(
+ final int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int fastSerializeScale,
+ long[] scratchLongs, byte[] buffer) {
+ if (fastSerializeScale != -1) {
+ return
+ fastBigIntegerBytesScaled(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ fastSerializeScale,
+ scratchLongs, buffer);
+ } else {
+ return
+ fastBigIntegerBytesUnscaled(
+ fastSignum, fast0, fast1, fast2,
+ scratchLongs, buffer);
+ }
+ }
+
+ /**
+ * Return binary representation of this decimal's BigInteger equivalent unscaled value using
+ * the format that the BigInteger's toByteArray method returns (and the BigInteger constructor
+ * accepts).
+ *
+ * Used by LazyBinary, Avro, and Parquet serialization.
+ *
+ * Scratch objects necessary to do the decimal to binary conversion without actually creating a
+ * BigInteger object are passed for better performance.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scratchLongs scratch array of SCRATCH_LONGS_LEN longs
+ * @param buffer scratch array of SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes
+ * @return The number of bytes used for the binary result in buffer. Otherwise, 0 if the
+ * conversion failed.
+ */
+ public static int fastBigIntegerBytesUnscaled(
+ final int fastSignum, long fast0, long fast1, long fast2,
+ long[] scratchLongs, byte[] buffer) {
+
+ /*
+ * Algorithm:
+ * 1) Convert decimal to three 56-bit words (three is enough for the decimal since we
+ * represent the decimal with trailing zeroes trimmed).
+ * 2) Skip leading zeroes in the words.
+ * 3) Once we find real data (i.e. a non-zero byte), add a sign byte to buffer if necessary.
+ * 4) Add bytes from the (rest of) 56-bit words.
+ * 5) Return byte count.
+ */
+
+ if (fastSignum == 0) {
+ buffer[0] = 0;
+ return 1;
+ }
+
+ boolean isNegative = (fastSignum == -1);
+
+ /*
+ * Use common conversion method we share with fastSerializationUtilsWrite.
+ */
+ if (!doDecimalToBinaryConversion(
+ fast0, fast1, fast2,
+ FAST_HIVE_DECIMAL_TWO_POWER_56_INVERSE,
+ BIG_INTEGER_BYTES_QUOTIENT_INTEGER_WORD_NUM,
+ BIG_INTEGER_BYTES_QUOTIENT_INTEGER_DIGIT_NUM,
+ FAST_HIVE_DECIMAL_TWO_POWER_56,
+ scratchLongs)) {
+ // Overflow. This is not expected.
+ return 0;
+ }
+
+ int byteIndex = 0;
+
+ long word0 = scratchLongs[0];
+ long word1 = scratchLongs[1];
+ long word2 = scratchLongs[2];
+
+ if (!isNegative) {
+
+ // Positive number.
+
+ long longWork = 0;
+
+ int shift = INITIAL_SHIFT;
+
+ if (word2 != 0L) {
+
+ // Skip leading zeroes in word2.
+
+ while (true) {
+ longWork = (word2 >> shift) & LONG_BYTE_MASK;
+ if (longWork != 0) {
+ break;
+ }
+ if (shift == 0) {
+ throw new RuntimeException("Unexpected #1");
+ }
+ shift -= Byte.SIZE;
+ }
+
+ // Now that we have found real data, emit sign byte if necessary.
+ if ((longWork & LONG_BYTE_HIGH_BIT_MASK) != 0) {
+ // Add sign byte since high bit is on.
+ buffer[byteIndex++] = (byte) 0;
+ }
+
+ // Emit the rest of word2
+ while (true) {
+ buffer[byteIndex++] = (byte) longWork;
+ if (shift == 0) {
+ break;
+ }
+ shift -= Byte.SIZE;
+ longWork = (word2 >> shift) & LONG_BYTE_MASK;
+ }
+
+ shift = INITIAL_SHIFT;
+ }
+
+ if (byteIndex == 0 && word1 == 0L) {
+
+ // Skip word1, also.
+
+ } else {
+
+ if (byteIndex == 0) {
+
+ // Skip leading zeroes in word1.
+
+ while (true) {
+ longWork = (word1 >> shift) & LONG_BYTE_MASK;
+ if (longWork != 0) {
+ break;
+ }
+ if (shift == 0) {
+ throw new RuntimeException("Unexpected #2");
+ }
+ shift -= Byte.SIZE;
+ }
+
+ // Now that we have found real data, emit sign byte if necessary.
+ if ((longWork & LONG_BYTE_HIGH_BIT_MASK) != 0) {
+ // Add sign byte since high bit is on.
+ buffer[byteIndex++] = (byte) 0;
+ }
+
+ } else {
+ longWork = (word1 >> shift) & LONG_BYTE_MASK;
+ }
+
+ // Emit the rest of word1
+
+ while (true) {
+ buffer[byteIndex++] = (byte) longWork;
+ if (shift == 0) {
+ break;
+ }
+ shift -= Byte.SIZE;
+ longWork = (word1 >> shift) & LONG_BYTE_MASK;
+ }
+
+ shift = INITIAL_SHIFT;
+ }
+
+ if (byteIndex == 0) {
+
+ // Skip leading zeroes in word0.
+
+ while (true) {
+ longWork = (word0 >> shift) & LONG_BYTE_MASK;
+ if (longWork != 0) {
+ break;
+ }
+ if (shift == 0) {
+
+ // All zeroes -- we should have handled this earlier.
+ throw new RuntimeException("Unexpected #3");
+ }
+ shift -= Byte.SIZE;
+ }
+
+ // Now that we have found real data, emit sign byte if necessary.
+ if ((longWork & LONG_BYTE_HIGH_BIT_MASK) != 0) {
+ // Add sign byte since high bit is on.
+ buffer[byteIndex++] = (byte) 0;
+ }
+
+ } else {
+ longWork = (word0 >> shift) & LONG_BYTE_MASK;
+ }
+
+ // Emit the rest of word0.
+ while (true) {
+ buffer[byteIndex++] = (byte) longWork;
+ if (shift == 0) {
+ break;
+ }
+ shift -= Byte.SIZE;
+ longWork = (word0 >> shift) & LONG_BYTE_MASK;
+ }
+
+ } else {
+
+ // Negative number.
+
+ // Subtract 1 for two's compliment adjustment.
+ word0--;
+ if (word0 < 0) {
+ word0 += LONG_TWO_TO_56_POWER;
+ word1--;
+ if (word1 < 0) {
+ word1 += LONG_TWO_TO_56_POWER;
+ word2--;
+ if (word2 < 0) {
+ // Underflow.
+ return 0;
+ }
+ }
+ }
+
+ long longWork = 0;
+
+ int shift = INITIAL_SHIFT;
+
+ if (word2 != 0L) {
+
+ // Skip leading zeroes in word2.
+
+ while (true) {
+ longWork = (word2 >> shift) & LONG_BYTE_MASK;
+ if (longWork != 0) {
+ break;
+ }
+ if (shift == 0) {
+ throw new RuntimeException("Unexpected #1");
+ }
+ shift -= Byte.SIZE;
+ }
+
+ // Now that we have found real data, emit sign byte if necessary and do negative fixup.
+
+ longWork = (~longWork & LONG_BYTE_MASK);
+ if (((longWork) & LONG_BYTE_HIGH_BIT_MASK) == 0) {
+ // Add sign byte since high bit is off.
+ buffer[byteIndex++] = BYTE_ALL_BITS;
+ }
+
+ // Invert words.
+ word2 = ~word2;
+ word1 = ~word1;
+ word0 = ~word0;
+
+ // Emit the rest of word2
+ while (true) {
+ buffer[byteIndex++] = (byte) longWork;
+ if (shift == 0) {
+ break;
+ }
+ shift -= Byte.SIZE;
+ longWork = (word2 >> shift) & LONG_BYTE_MASK;
+ }
+
+ shift = INITIAL_SHIFT;
+ }
+
+ if (byteIndex == 0 && word1 == 0L) {
+
+ // Skip word1, also.
+
+ } else {
+
+ if (byteIndex == 0) {
+
+ // Skip leading zeroes in word1.
+
+ while (true) {
+ longWork = (word1 >> shift) & LONG_BYTE_MASK;
+ if (longWork != 0) {
+ break;
+ }
+ if (shift == 0) {
+ throw new RuntimeException("Unexpected #2");
+ }
+ shift -= Byte.SIZE;
+ }
+
+ // Now that we have found real data, emit sign byte if necessary and do negative fixup.
+
+ longWork = (~longWork & LONG_BYTE_MASK);
+ if ((longWork & LONG_BYTE_HIGH_BIT_MASK) == 0) {
+ // Add sign byte since high bit is off.
+ buffer[byteIndex++] = BYTE_ALL_BITS;
+ }
+
+ // Invert words.
+ word1 = ~word1;
+ word0 = ~word0;
+
+ } else {
+ longWork = (word1 >> shift) & LONG_BYTE_MASK;
+ }
+
+ // Emit the rest of word1
+
+ while (true) {
+ buffer[byteIndex++] = (byte) longWork;
+ if (shift == 0) {
+ break;
+ }
+ shift -= Byte.SIZE;
+ longWork = (word1 >> shift) & LONG_BYTE_MASK;
+ }
+
+ shift = INITIAL_SHIFT;
+ }
+
+ if (byteIndex == 0) {
+
+ // Skip leading zeroes in word0.
+
+ while (true) {
+ longWork = (word0 >> shift) & LONG_BYTE_MASK;
+ if (longWork != 0) {
+ break;
+ }
+ if (shift == 0) {
+
+ // All zeroes.
+
+ // -1 special case. Unsigned magnitude 1 - two's compliment adjustment 1 = 0.
+ buffer[0] = BYTE_ALL_BITS;
+ return 1;
+ }
+ shift -= Byte.SIZE;
+ }
+
+ // Now that we have found real data, emit sign byte if necessary and do negative fixup.
+
+ longWork = (~longWork & LONG_BYTE_MASK);
+ if ((longWork & LONG_BYTE_HIGH_BIT_MASK) == 0) {
+ // Add sign byte since high bit is off.
+ buffer[byteIndex++] = BYTE_ALL_BITS;
+ }
+
+ // Invert words.
+ word0 = ~word0;
+
+ } else {
+ longWork = (word0 >> shift) & LONG_BYTE_MASK;
+ }
+
+ // Emit the rest of word0.
+ while (true) {
+ buffer[byteIndex++] = (byte) longWork;
+ if (shift == 0) {
+ break;
+ }
+ shift -= Byte.SIZE;
+ longWork = (word0 >> shift) & LONG_BYTE_MASK;
+ }
+ }
+
+ return byteIndex;
+ }
+
+ /**
+ * Convert decimal to BigInteger binary bytes with a serialize scale, similar to the formatScale
+ * for toFormatString. It adds trailing zeroes when a serializeScale is greater than current
+ * scale. Or, rounds if scale is less than current scale.
+ *
+ * Used by Avro and Parquet serialization.
+ *
+ * This emulates the OldHiveDecimal setScale / OldHiveDecimal getInternalStorage() behavior.
+ *
+ * @param fastSignum the sign number (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale
+ * @param serializeScale the scale to serialize
+ * @param scratchLongs a scratch array of longs
+ * @param buffer the buffer to serialize into
+ * @return the number of bytes used to serialize the number
+ */
+ public static int fastBigIntegerBytesScaled(
+ final int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int serializeScale,
+ long[] scratchLongs, byte[] buffer) {
+
+ // Normally, trailing fractional digits are removed. But to emulate the
+ // OldHiveDecimal setScale and OldHiveDecimalWritable internalStorage, we need to trailing zeroes
+ // here.
+ //
+ // NOTE: This can cause a decimal that has too many decimal digits (because of trailing zeroes)
+ // for us to represent. In that case, we punt and convert with a BigInteger alternate
+ // code.
+
+ if (fastSignum == 0 || serializeScale == fastScale) {
+ return
+ fastBigIntegerBytesUnscaled(
+ fastSignum, fast0, fast1, fast2,
+ scratchLongs, buffer);
+ } else if (serializeScale > fastScale) {
+
+ final int scaleUp = serializeScale - fastScale;
+ final int maxScale = HiveDecimal.MAX_SCALE - fastIntegerDigitCount;
+ if (serializeScale > maxScale) {
+
+ // We cannot to scaled up decimals that cannot be represented.
+ // Instead, we use a BigInteger instead.
+
+ BigInteger bigInteger =
+ fastBigIntegerValueUnscaled(
+ fastSignum, fast0, fast1, fast2);
+
+ BigInteger bigIntegerScaled = bigInteger.multiply(BIG_INTEGER_TEN.pow(scaleUp));
+ byte[] bigIntegerBytesScaled = bigIntegerScaled.toByteArray();
+ final int length = bigIntegerBytesScaled.length;
+ System.arraycopy(bigIntegerBytesScaled, 0, buffer, 0, length);
+ return length;
+ }
+
+ FastHiveDecimal fastTemp = new FastHiveDecimal();
+ if (!fastScaleUp(
+ fast0, fast1, fast2,
+ scaleUp,
+ fastTemp)) {
+ throw new RuntimeException("Unexpected");
+ }
+ return
+ fastBigIntegerBytesUnscaled(
+ fastSignum, fastTemp.fast0, fastTemp.fast1, fastTemp.fast2,
+ scratchLongs, buffer);
+ } else {
+
+ // serializeScale < fastScale.
+
+ FastHiveDecimal fastTemp = new FastHiveDecimal();
+ if (!fastRound(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ serializeScale, BigDecimal.ROUND_HALF_UP,
+ fastTemp)) {
+ return 0;
+ }
+ return
+ fastBigIntegerBytesUnscaled(
+ fastSignum, fastTemp.fast0, fastTemp.fast1, fastTemp.fast2,
+ scratchLongs, buffer);
+ }
+ }
+
+ //************************************************************************************************
+ // Decimal to Integer conversion.
+
+ private static final int MAX_BYTE_DIGITS = 3;
+ private static final FastHiveDecimal FASTHIVEDECIMAL_MIN_BYTE_VALUE_MINUS_ONE =
+ new FastHiveDecimal((long) Byte.MIN_VALUE - 1L);
+ private static final FastHiveDecimal FASTHIVEDECIMAL_MAX_BYTE_VALUE_PLUS_ONE =
+ new FastHiveDecimal((long) Byte.MAX_VALUE + 1L);
+
+ private static final int MAX_SHORT_DIGITS = 5;
+ private static final FastHiveDecimal FASTHIVEDECIMAL_MIN_SHORT_VALUE_MINUS_ONE =
+ new FastHiveDecimal((long) Short.MIN_VALUE - 1L);
+ private static final FastHiveDecimal FASTHIVEDECIMAL_MAX_SHORT_VALUE_PLUS_ONE =
+ new FastHiveDecimal((long) Short.MAX_VALUE + 1L);
+
+ private static final int MAX_INT_DIGITS = 10;
+ private static final FastHiveDecimal FASTHIVEDECIMAL_MIN_INT_VALUE_MINUS_ONE =
+ new FastHiveDecimal((long) Integer.MIN_VALUE - 1L);
+ private static final FastHiveDecimal FASTHIVEDECIMAL_MAX_INT_VALUE_PLUS_ONE =
+ new FastHiveDecimal((long) Integer.MAX_VALUE + 1L);
+
+ private static final FastHiveDecimal FASTHIVEDECIMAL_MIN_LONG_VALUE =
+ new FastHiveDecimal(Long.MIN_VALUE);
+ private static final FastHiveDecimal FASTHIVEDECIMAL_MAX_LONG_VALUE =
+ new FastHiveDecimal(Long.MAX_VALUE);
+ private static final int MAX_LONG_DIGITS =
+ FASTHIVEDECIMAL_MAX_LONG_VALUE.fastIntegerDigitCount;
+ private static final FastHiveDecimal FASTHIVEDECIMAL_MIN_LONG_VALUE_MINUS_ONE =
+ new FastHiveDecimal("-9223372036854775809");
+ private static final FastHiveDecimal FASTHIVEDECIMAL_MAX_LONG_VALUE_PLUS_ONE =
+ new FastHiveDecimal("9223372036854775808");
+
+ private static final BigInteger BIG_INTEGER_UNSIGNED_BYTE_MAX_VALUE = BIG_INTEGER_TWO.pow(Byte.SIZE).subtract(BigInteger.ONE);
+ private static final BigInteger BIG_INTEGER_UNSIGNED_SHORT_MAX_VALUE = BIG_INTEGER_TWO.pow(Short.SIZE).subtract(BigInteger.ONE);
+ private static final BigInteger BIG_INTEGER_UNSIGNED_INT_MAX_VALUE = BIG_INTEGER_TWO.pow(Integer.SIZE).subtract(BigInteger.ONE);
+ private static final BigInteger BIG_INTEGER_UNSIGNED_LONG_MAX_VALUE = BIG_INTEGER_TWO.pow(Long.SIZE).subtract(BigInteger.ONE);
+
+ /**
+ * Is the decimal value a byte? Range -128 to 127.
+ * Byte.MIN_VALUE Byte.MAX_VALUE
+ *
+ * Emulates testing for no value corruption:
+ * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().byteValue()))
+ *
+ * NOTE: Fractional digits are ignored in the test since fastByteValueClip() will
+ * remove them (round down).
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @return True when fastByteValueClip() will return a correct byte.
+ */
+ public static boolean fastIsByte(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+
+ if (fastIntegerDigitCount < MAX_BYTE_DIGITS) {
+
+ // Definitely a byte; most bytes fall here
+ return true;
+
+ } else if (fastIntegerDigitCount > MAX_BYTE_DIGITS) {
+
+ // Definitely not a byte.
+ return false;
+
+ } else if (fastScale == 0) {
+ if (fast1 != 0 || fast2 != 0) {
+ return false;
+ }
+ if (fastSignum == 1) {
+ return (fast0 <= Byte.MAX_VALUE);
+ } else {
+ return (-fast0 >= Byte.MIN_VALUE);
+ }
+ } else {
+
+ // We need to work a little harder for our comparison. Note we round down for
+ // integer conversion so anything below the next min/max will work.
+
+ if (fastSignum == 1) {
+ return
+ (fastCompareTo(
+ fastSignum, fast0, fast1, fast2, fastScale,
+ FASTHIVEDECIMAL_MAX_BYTE_VALUE_PLUS_ONE) < 0);
+ } else {
+ return
+ (fastCompareTo(
+ fastSignum, fast0, fast1, fast2, fastScale,
+ FASTHIVEDECIMAL_MIN_BYTE_VALUE_MINUS_ONE) > 0);
+ }
+ }
+ }
+
+ // We use "Clip" in the name because this method will return a corrupted value when
+ // fastIsByte returns false.
+ public static byte fastByteValueClip(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+
+ if (fastScale == 0) {
+ if (fast1 == 0 && fast2 == 0) {
+ if (fastSignum == 1) {
+ if (fast0 <= Byte.MAX_VALUE) {
+ return (byte) fast0;
+ }
+ } else {
+ if (-fast0 >= Byte.MIN_VALUE) {
+ return (byte) -fast0;
+ };
+ }
+ }
+ // SLOW: Do remainder with BigInteger.
+ BigInteger bigInteger =
+ fastBigIntegerValueUnscaled(
+ fastSignum, fast0, fast1, fast2);
+ return bigInteger.remainder(BIG_INTEGER_UNSIGNED_BYTE_MAX_VALUE).byteValue();
+ } else {
+
+ // Adjust all longs using power 10 division/remainder.
+ long result0;
+ long result1;
+ long result2;
+ if (fastScale < LONGWORD_DECIMAL_DIGITS) {
+
+ // Part of lowest word survives.
+
+ final long divideFactor = powerOfTenTable[fastScale];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale];
+
+ result0 =
+ fast0 / divideFactor
+ + ((fast1 % divideFactor) * multiplyFactor);
+ result1 =
+ fast1 / divideFactor
+ + ((fast2 % divideFactor) * multiplyFactor);
+ result2 =
+ fast2 / divideFactor;
+
+ } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ // Throw away lowest word.
+
+ final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS;
+
+ final long divideFactor = powerOfTenTable[adjustedScaleDown];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown];
+
+ result0 =
+ fast1 / divideFactor
+ + ((fast2 % divideFactor) * multiplyFactor);
+ result1 =
+ fast2 / divideFactor;
+ result2 = 0;
+
+ } else {
+
+ // Throw away middle and lowest words.
+
+ final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS;
+
+ result0 =
+ fast2 / powerOfTenTable[adjustedScaleDown];
+ result1 = 0;
+ result2 = 0;
+
+ }
+
+ if (result1 == 0 && result2 == 0) {
+ if (fastSignum == 1) {
+ if (result0 <= Byte.MAX_VALUE) {
+ return (byte) result0;
+ }
+ } else {
+ if (-result0 >= Byte.MIN_VALUE) {
+ return (byte) -result0;
+ };
+ }
+ }
+ // SLOW: Do remainder with BigInteger.
+ BigInteger bigInteger =
+ fastBigIntegerValueUnscaled(
+ fastSignum, result0, result1, result2);
+ return bigInteger.remainder(BIG_INTEGER_UNSIGNED_BYTE_MAX_VALUE).byteValue();
+ }
+ }
+
+ /**
+
+ * @return True when shortValue() will return a correct short.
+ */
+
+ /**
+ * Is the decimal value a short? Range -32,768 to 32,767.
+ * Short.MIN_VALUE Short.MAX_VALUE
+ *
+ * Emulates testing for no value corruption:
+ * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().shortValue()))
+ *
+ * NOTE: Fractional digits are ignored in the test since fastShortValueClip() will
+ * remove them (round down).
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @return True when fastShortValueClip() will return a correct short.
+ */
+ public static boolean fastIsShort(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+
+ if (fastIntegerDigitCount < MAX_SHORT_DIGITS) {
+
+ // Definitely a short; most shorts fall here
+ return true;
+
+ } else if (fastIntegerDigitCount > MAX_SHORT_DIGITS) {
+
+ // Definitely not a short.
+ return false;
+
+ } else if (fastScale == 0) {
+ if (fast1 != 0 || fast2 != 0) {
+ return false;
+ }
+ if (fastSignum == 1) {
+ return (fast0 <= Short.MAX_VALUE);
+ } else {
+ return (-fast0 >= Short.MIN_VALUE);
+ }
+ } else {
+
+ // We need to work a little harder for our comparison. Note we round down for
+ // integer conversion so anything below the next min/max will work.
+
+ if (fastSignum == 1) {
+ return
+ (fastCompareTo(
+ fastSignum, fast0, fast1, fast2, fastScale,
+ FASTHIVEDECIMAL_MAX_SHORT_VALUE_PLUS_ONE) < 0);
+ } else {
+ return
+ (fastCompareTo(
+ fastSignum, fast0, fast1, fast2, fastScale,
+ FASTHIVEDECIMAL_MIN_SHORT_VALUE_MINUS_ONE) > 0);
+ }
+ }
+ }
+
+ // We use "Clip" in the name because this method will return a corrupted value when
+ // fastIsShort returns false.
+ public static short fastShortValueClip(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+
+ if (fastScale == 0) {
+ if (fast1 == 0 && fast2 == 0) {
+ if (fastSignum == 1) {
+ if (fast0 <= Short.MAX_VALUE) {
+ return (short) fast0;
+ }
+ } else {
+ if (-fast0 >= Short.MIN_VALUE) {
+ return (short) -fast0;
+ };
+ }
+ }
+ // SLOW: Do remainder with BigInteger.
+ BigInteger bigInteger =
+ fastBigIntegerValueUnscaled(
+ fastSignum, fast0, fast1, fast2);
+ return bigInteger.remainder(BIG_INTEGER_UNSIGNED_SHORT_MAX_VALUE).shortValue();
+ } else {
+
+ // Adjust all longs using power 10 division/remainder.
+ long result0;
+ long result1;
+ long result2;
+ if (fastScale < LONGWORD_DECIMAL_DIGITS) {
+
+ // Part of lowest word survives.
+
+ final long divideFactor = powerOfTenTable[fastScale];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale];
+
+ result0 =
+ fast0 / divideFactor
+ + ((fast1 % divideFactor) * multiplyFactor);
+ result1 =
+ fast1 / divideFactor
+ + ((fast2 % divideFactor) * multiplyFactor);
+ result2 =
+ fast2 / divideFactor;
+
+ } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ // Throw away lowest word.
+
+ final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS;
+
+ final long divideFactor = powerOfTenTable[adjustedScaleDown];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown];
+
+ result0 =
+ fast1 / divideFactor
+ + ((fast2 % divideFactor) * multiplyFactor);
+ result1 =
+ fast2 / divideFactor;
+ result2 = 0;
+
+ } else {
+
+ // Throw away middle and lowest words.
+
+ final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS;
+
+ result0 =
+ fast2 / powerOfTenTable[adjustedScaleDown];
+ result1 = 0;
+ result2 = 0;
+
+ }
+
+ if (result1 == 0 && result2 == 0) {
+ if (fastSignum == 1) {
+ if (result0 <= Short.MAX_VALUE) {
+ return (short) result0;
+ }
+ } else {
+ if (-result0 >= Short.MIN_VALUE) {
+ return (short) -result0;
+ };
+ }
+ }
+ // SLOW: Do remainder with BigInteger.
+ BigInteger bigInteger =
+ fastBigIntegerValueUnscaled(
+ fastSignum, result0, result1, result2);
+ return bigInteger.remainder(BIG_INTEGER_UNSIGNED_SHORT_MAX_VALUE).shortValue();
+ }
+ }
+
+ /**
+ * Is the decimal value a int? Range -2,147,483,648 to 2,147,483,647.
+ * Integer.MIN_VALUE Integer.MAX_VALUE
+ *
+ * Emulates testing for no value corruption:
+ * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().intValue()))
+ *
+ * NOTE: Fractional digits are ignored in the test since fastIntValueClip() will
+ * remove them (round down).
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @return True when fastIntValueClip() will return a correct int.
+ */
+ public static boolean fastIsInt(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+
+ if (fastIntegerDigitCount < MAX_INT_DIGITS) {
+
+ // Definitely a int; most ints fall here
+ return true;
+
+ } else if (fastIntegerDigitCount > MAX_INT_DIGITS) {
+
+ // Definitely not an int.
+ return false;
+
+ } else if (fastScale == 0) {
+ if (fast1 != 0 || fast2 != 0) {
+ return false;
+ }
+ if (fastSignum == 1) {
+ return (fast0 <= Integer.MAX_VALUE);
+ } else {
+ return (-fast0 >= Integer.MIN_VALUE);
+ }
+ } else {
+
+ // We need to work a little harder for our comparison. Note we round down for
+ // integer conversion so anything below the next min/max will work.
+
+ if (fastSignum == 1) {
+ return
+ (fastCompareTo(
+ fastSignum, fast0, fast1, fast2, fastScale,
+ FASTHIVEDECIMAL_MAX_INT_VALUE_PLUS_ONE) < 0);
+ } else {
+ return
+ (fastCompareTo(
+ fastSignum, fast0, fast1, fast2, fastScale,
+ FASTHIVEDECIMAL_MIN_INT_VALUE_MINUS_ONE) > 0);
+ }
+ }
+ }
+
+ // We use "Clip" in the name because this method will return a corrupted value when
+ // fastIsInt returns false.
+ public static int fastIntValueClip(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+
+ if (fastScale == 0) {
+ if (fast1 == 0 && fast2 == 0) {
+ if (fastSignum == 1) {
+ if (fast0 <= Integer.MAX_VALUE) {
+ return (int) fast0;
+ }
+ } else {
+ if (-fast0 >= Integer.MIN_VALUE) {
+ return (int) -fast0;
+ };
+ }
+ }
+ // SLOW: Do remainder with BigInteger.
+ BigInteger bigInteger =
+ fastBigIntegerValueUnscaled(
+ fastSignum, fast0, fast1, fast2);
+ return bigInteger.remainder(BIG_INTEGER_UNSIGNED_INT_MAX_VALUE).intValue();
+ } else {
+
+ // Adjust all longs using power 10 division/remainder.
+ long result0;
+ long result1;
+ long result2;
+ if (fastScale < LONGWORD_DECIMAL_DIGITS) {
+
+ // Part of lowest word survives.
+
+ final long divideFactor = powerOfTenTable[fastScale];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale];
+
+ result0 =
+ fast0 / divideFactor
+ + ((fast1 % divideFactor) * multiplyFactor);
+ result1 =
+ fast1 / divideFactor
+ + ((fast2 % divideFactor) * multiplyFactor);
+ result2 =
+ fast2 / divideFactor;
+
+ } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ // Throw away lowest word.
+
+ final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS;
+
+ final long divideFactor = powerOfTenTable[adjustedScaleDown];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown];
+
+ result0 =
+ fast1 / divideFactor
+ + ((fast2 % divideFactor) * multiplyFactor);
+ result1 =
+ fast2 / divideFactor;
+ result2 = 0;
+
+ } else {
+
+ // Throw away middle and lowest words.
+
+ final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS;
+
+ result0 =
+ fast2 / powerOfTenTable[adjustedScaleDown];
+ result1 = 0;
+ result2 = 0;
+
+ }
+
+ if (result1 == 0 && result2 == 0) {
+ if (fastSignum == 1) {
+ if (result0 <= Integer.MAX_VALUE) {
+ return (int) result0;
+ }
+ } else {
+ if (-result0 >= Integer.MIN_VALUE) {
+ return (int) -result0;
+ };
+ }
+ }
+ // SLOW: Do remainder with BigInteger.
+ BigInteger bigInteger =
+ fastBigIntegerValueUnscaled(
+ fastSignum, result0, result1, result2);
+ return bigInteger.remainder(BIG_INTEGER_UNSIGNED_INT_MAX_VALUE).intValue();
+ }
+ }
+
+ /**
+ * Is the decimal value a long? Range -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807.
+ * Long.MIN_VALUE Long.MAX_VALUE
+ *
+ * Emulates testing for no value corruption:
+ * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().longValue()))
+ *
+ * NOTE: Fractional digits are ignored in the test since fastLongValueClip() will
+ * remove them (round down).
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @return True when fastLongValueClip() will return a correct long.
+ */
+ public static boolean fastIsLong(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+
+ if (fastIntegerDigitCount < MAX_LONG_DIGITS) {
+
+ // Definitely a long; most longs fall here
+ return true;
+
+ } else if (fastIntegerDigitCount > MAX_LONG_DIGITS) {
+
+ // Definitely not a long.
+ return false;
+
+ } else if (fastScale == 0) {
+
+ // From the above checks, we know fast2 is zero.
+
+ if (fastSignum == 1) {
+ FastHiveDecimal max = FASTHIVEDECIMAL_MAX_LONG_VALUE;
+ if (fast1 > max.fast1 || (fast1 == max.fast1 && fast0 > max.fast0)) {
+ return false;
+ }
+ return true;
+ } else {
+ FastHiveDecimal min = FASTHIVEDECIMAL_MIN_LONG_VALUE;
+ if (fast1 > min.fast1 || (fast1 == min.fast1 && fast0 > min.fast0)) {
+ return false;
+ }
+ return true;
+ }
+
+ } else {
+
+ // We need to work a little harder for our comparison. Note we round down for
+ // integer conversion so anything below the next min/max will work.
+
+ if (fastSignum == 1) {
+ return
+ (fastCompareTo(
+ fastSignum, fast0, fast1, fast2, fastScale,
+ FASTHIVEDECIMAL_MAX_LONG_VALUE_PLUS_ONE) < 0);
+ } else {
+ return
+ (fastCompareTo(
+ fastSignum, fast0, fast1, fast2, fastScale,
+ FASTHIVEDECIMAL_MIN_LONG_VALUE_MINUS_ONE) > 0);
+ }
+ }
+ }
+
+ // We use "Clip" in the name because this method will return a corrupted value when
+ // fastIsLong returns false.
+ public static long fastLongValueClip(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+ if (fastSignum == 0) {
+ return 0;
+ }
+
+ if (fastScale == 0) {
+ // Do first comparison as unsigned.
+ if (fastCompareTo(
+ 1, fast0, fast1, fast2, fastScale,
+ FASTHIVEDECIMAL_MAX_LONG_VALUE) <= 0) {
+ if (fastSignum == 1) {
+ return
+ fast1 * MULTIPLER_LONGWORD_DECIMAL
+ + fast0;
+ } else {
+ return
+ -(fast1 * MULTIPLER_LONGWORD_DECIMAL
+ + fast0);
+ }
+ } if (fastEquals(
+ fastSignum, fast0, fast1, fast2, fastScale,
+ FASTHIVEDECIMAL_MIN_LONG_VALUE)) {
+ return Long.MIN_VALUE;
+ } else {
+ // SLOW: Do remainder with BigInteger.
+ BigInteger bigInteger =
+ fastBigIntegerValueUnscaled(
+ fastSignum, fast0, fast1, fast2);
+ return bigInteger.remainder(BIG_INTEGER_UNSIGNED_LONG_MAX_VALUE).longValue();
+ }
+ } else {
+
+ // Adjust all longs using power 10 division/remainder.
+ long result0;
+ long result1;
+ long result2;
+ if (fastScale < LONGWORD_DECIMAL_DIGITS) {
+
+ // Part of lowest word survives.
+
+ final long divideFactor = powerOfTenTable[fastScale];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale];
+
+ result0 =
+ fast0 / divideFactor
+ + ((fast1 % divideFactor) * multiplyFactor);
+ result1 =
+ fast1 / divideFactor
+ + ((fast2 % divideFactor) * multiplyFactor);
+ result2 =
+ fast2 / divideFactor;
+
+ } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ // Throw away lowest word.
+
+ final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS;
+
+ final long divideFactor = powerOfTenTable[adjustedScaleDown];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown];
+
+ result0 =
+ fast1 / divideFactor
+ + ((fast2 % divideFactor) * multiplyFactor);
+ result1 =
+ fast2 / divideFactor;
+ result2 = 0;
+
+ } else {
+
+ // Throw away middle and lowest words.
+
+ final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS;
+
+ result0 =
+ fast2 / powerOfTenTable[adjustedScaleDown];
+ result1 = 0;
+ result2 = 0;
+
+ }
+
+ // Do first comparison as UNSIGNED.
+ if (fastCompareTo(
+ 1, result0, result1, result2, /* fastScale */ 0,
+ FASTHIVEDECIMAL_MAX_LONG_VALUE) <= 0) {
+ if (fastSignum == 1) {
+ return
+ result1 * MULTIPLER_LONGWORD_DECIMAL
+ + result0;
+ } else {
+ return
+ -(result1 * MULTIPLER_LONGWORD_DECIMAL
+ + result0);
+ }
+ } if (fastEquals(
+ fastSignum, result0, result1, result2, /* fastScale */ 0,
+ FASTHIVEDECIMAL_MIN_LONG_VALUE)) {
+
+ // SIGNED comparison to Long.MIN_VALUE decimal.
+ return Long.MIN_VALUE;
+ } else {
+ // SLOW: Do remainder with BigInteger.
+ BigInteger bigInteger =
+ fastBigIntegerValueUnscaled(
+ fastSignum, result0, result1, result2);
+ return bigInteger.remainder(BIG_INTEGER_UNSIGNED_LONG_MAX_VALUE).longValue();
+ }
+ }
+ }
+
+ //************************************************************************************************
+ // Decimal to Non-Integer conversion.
+
+ public static float fastFloatValue(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+ if (fastSignum == 0) {
+ return 0;
+ }
+ BigDecimal bigDecimal = fastBigDecimalValue(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale);
+ return bigDecimal.floatValue();
+ }
+
+ public static double fastDoubleValue(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+ if (fastSignum == 0) {
+ return 0;
+ }
+
+ // CONSIDER: Looked at the possibility of faster decimal to double conversion by using some
+ // of their lower level logic that extracts the various parts out of a double.
+ // The difficulty is Java's rounding rules are byzantine.
+
+ BigDecimal bigDecimal = fastBigDecimalValue(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale);
+ return bigDecimal.doubleValue();
+ }
+
+ /**
+ * Get a BigInteger representing the decimal's digits without a dot.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @param fastSerializationScale the scale to serialize
+ * @return Returns a signed BigInteger.
+ */
+ public static BigInteger fastBigIntegerValue(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int fastSerializationScale) {
+ if (fastSerializationScale != -1) {
+ return
+ fastBigIntegerValueScaled(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ fastSerializationScale);
+ } else {
+ return
+ fastBigIntegerValueUnscaled(
+ fastSignum, fast0, fast1, fast2);
+ }
+ }
+
+ public static BigInteger fastBigIntegerValueUnscaled(
+ int fastSignum, long fast0, long fast1, long fast2) {
+
+ if (fastSignum == 0) {
+ return BigInteger.ZERO;
+ }
+ BigInteger result;
+ if (fast2 == 0) {
+ if (fast1 == 0) {
+ result =
+ BigInteger.valueOf(fast0);
+ } else {
+ result =
+ BigInteger.valueOf(fast0).add(
+ BigInteger.valueOf(fast1).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER));
+ }
+ } else {
+ result =
+ BigInteger.valueOf(fast0).add(
+ BigInteger.valueOf(fast1).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER)).add(
+ BigInteger.valueOf(fast2).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER_2X));
+ }
+
+ return (fastSignum == 1 ? result : result.negate());
+ }
+
+ public static BigInteger fastBigIntegerValueScaled(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int fastSerializationScale) {
+
+ // Use the serialization scale and create a BigInteger with trailing zeroes (or
+ // round the decimal) if necessary.
+ //
+ // Since we are emulating old behavior and recommending the use of HiveDecimal.bigIntegerBytesScaled
+ // instead just do it the slow way. Get the BigDecimal.setScale value and return the
+ // BigInteger.
+ //
+ BigDecimal bigDecimal =
+ fastBigDecimalValue(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale);
+ bigDecimal = bigDecimal.setScale(fastSerializationScale, RoundingMode.HALF_UP);
+ return bigDecimal.unscaledValue();
+ }
+
+ /**
+ * Return a BigDecimal representing the decimal. The BigDecimal class is able to accurately
+ * represent the decimal.
+ *
+ * NOTE: We are not representing our decimal as BigDecimal now as OldHiveDecimal did, so this
+ * is now slower.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @return the BigDecimal equivalent
+ */
+ public static BigDecimal fastBigDecimalValue(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+ BigInteger unscaledValue =
+ fastBigIntegerValueUnscaled(
+ fastSignum, fast0, fast1, fast2);
+ return new BigDecimal(unscaledValue, fastScale);
+ }
+
+ //************************************************************************************************
+ // Decimal Comparison.
+
+ public static int fastCompareTo(
+ int leftSignum, long leftFast0, long leftFast1, long leftFast2,
+ int leftScale,
+ FastHiveDecimal fastRight) {
+
+ return
+ fastCompareTo(
+ leftSignum, leftFast0, leftFast1, leftFast2,
+ leftScale,
+ fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2,
+ fastRight.fastScale);
+ }
+
+ private static int doCompareToSameScale(
+ int signum,
+ long leftFast0, long leftFast1, long leftFast2,
+ long rightFast0, long rightFast1, long rightFast2) {
+
+ if (leftFast0 == rightFast0 && leftFast1 == rightFast1 && leftFast2 == rightFast2) {
+ return 0;
+ }
+ if (leftFast2 < rightFast2) {
+ return -signum;
+ } else if (leftFast2 > rightFast2) {
+ return signum;
+ }
+ if (leftFast1 < rightFast1) {
+ return -signum;
+ } else if (leftFast1 > rightFast1){
+ return signum;
+ }
+ return (leftFast0 < rightFast0 ? -signum : signum);
+ }
+
+ public static int fastCompareTo(
+ int leftSignum, long leftFast0, long leftFast1, long leftFast2,
+ int leftScale,
+ int rightSignum, long rightFast0, long rightFast1, long rightFast2,
+ int rightScale) {
+
+ if (leftSignum == 0 && rightSignum == 0) {
+ return 0;
+ }
+
+ // Optimization copied from BigDecimal.
+ int signDiff = leftSignum - rightSignum;
+ if (signDiff != 0) {
+ return (signDiff > 0 ? 1 : -1);
+ }
+
+ // We are here when the left and right are non-zero and have the same sign.
+
+ if (leftScale == rightScale) {
+
+ return doCompareToSameScale(
+ leftSignum,
+ leftFast0, leftFast1, leftFast2,
+ rightFast0, rightFast1, rightFast2);
+
+ } else {
+
+ // How do we handle different scales?
+
+ // We at least know they are not equal. The one with the larger scale has non-zero digits
+ // below the other's scale (since the scale does not include trailing zeroes).
+
+ // For comparison purposes, we can scale away those digits. And, we can not scale up since
+ // that could overflow.
+
+ // Use modified portions of doFastScaleDown code here since we do not want to allocate a
+ // temporary FastHiveDecimal object.
+
+ long compare0;
+ long compare1;
+ long compare2;
+ int scaleDown;
+ if (leftScale < rightScale) {
+
+ // Scale down right and compare.
+ scaleDown = rightScale - leftScale;
+
+ // Adjust all longs using power 10 division/remainder.
+
+ if (scaleDown < LONGWORD_DECIMAL_DIGITS) {
+ // Part of lowest word survives.
+
+ final long divideFactor = powerOfTenTable[scaleDown];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown];
+
+ compare0 =
+ rightFast0 / divideFactor
+ + ((rightFast1 % divideFactor) * multiplyFactor);
+ compare1 =
+ rightFast1 / divideFactor
+ + ((rightFast2 % divideFactor) * multiplyFactor);
+ compare2 =
+ rightFast2 / divideFactor;
+ } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+ // Throw away lowest word.
+
+ final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS;
+
+ final long divideFactor = powerOfTenTable[adjustedScaleDown];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown];
+
+ compare0 =
+ rightFast1 / divideFactor
+ + ((rightFast2 % divideFactor) * multiplyFactor);
+ compare1 =
+ rightFast2 / divideFactor;
+ compare2 = 0;
+ } else {
+ // Throw away middle and lowest words.
+
+ final int adjustedScaleDown = scaleDown - TWO_X_LONGWORD_DECIMAL_DIGITS;
+
+ compare0 =
+ rightFast2 / powerOfTenTable[adjustedScaleDown];
+ compare1 = 0;
+ compare2 = 0;
+ }
+
+ if (leftFast0 == compare0 && leftFast1 == compare1 && leftFast2 == compare2) {
+ // Return less than because of right's digits below left's scale.
+ return -leftSignum;
+ }
+ if (leftFast2 < compare2) {
+ return -leftSignum;
+ } else if (leftFast2 > compare2) {
+ return leftSignum;
+ }
+ if (leftFast1 < compare1) {
+ return -leftSignum;
+ } else if (leftFast1 > compare1){
+ return leftSignum;
+ }
+ return (leftFast0 < compare0 ? -leftSignum : leftSignum);
+
+ } else {
+
+ // Scale down left and compare.
+ scaleDown = leftScale - rightScale;
+
+ // Adjust all longs using power 10 division/remainder.
+
+ if (scaleDown < LONGWORD_DECIMAL_DIGITS) {
+ // Part of lowest word survives.
+
+ final long divideFactor = powerOfTenTable[scaleDown];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown];
+
+ compare1 =
+ leftFast1 / divideFactor
+ + ((leftFast2 % divideFactor) * multiplyFactor);
+ compare0 =
+ leftFast0 / divideFactor
+ + ((leftFast1 % divideFactor) * multiplyFactor);
+ compare2 =
+ leftFast2 / divideFactor;
+ } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+ // Throw away lowest word.
+
+ final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS;
+
+ final long divideFactor = powerOfTenTable[adjustedScaleDown];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown];
+
+ compare0 =
+ leftFast1 / divideFactor
+ + ((leftFast2 % divideFactor) * multiplyFactor);
+ compare1 =
+ leftFast2 / divideFactor;
+ compare2 = 0;
+ } else {
+ // Throw away middle and lowest words.
+
+ final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS;
+
+ compare0 =
+ leftFast2 / powerOfTenTable[adjustedScaleDown];
+ compare1 = 0;
+ compare2 = 0;
+ }
+
+ if (compare0 == rightFast0 && compare1 == rightFast1 && compare2 == rightFast2) {
+ // Return greater than because of left's digits below right's scale.
+ return leftSignum;
+ }
+ if (compare2 < rightFast2) {
+ return -leftSignum;
+ } else if (compare2 > rightFast2) {
+ return leftSignum;
+ }
+ if (compare1 < rightFast1) {
+ return -leftSignum;
+ } else if (compare1 > rightFast1){
+ return leftSignum;
+ }
+ return (compare0 < rightFast0 ? -leftSignum : leftSignum);
+
+ }
+ }
+ }
+
+ public static boolean fastEquals(
+ int leftSignum, long leftFast0, long leftFast1, long leftFast2,
+ int leftScale,
+ FastHiveDecimal fastRight) {
+
+ if (leftSignum == 0) {
+ return (fastRight.fastSignum == 0);
+ }
+ if (leftSignum != fastRight.fastSignum) {
+ return false;
+ }
+ if (leftScale != fastRight.fastScale) {
+ // We know they are not equal because the one with the larger scale has non-zero digits
+ // below the other's scale (since the scale does not include trailing zeroes).
+ return false;
+ }
+ return (
+ leftFast0 == fastRight.fast0 && leftFast1 == fastRight.fast1 && leftFast2 == fastRight.fast2);
+ }
+
+ public static boolean fastEquals(
+ int leftSignum, long leftFast0, long leftFast1, long leftFast2,
+ int leftScale,
+ int rightSignum, long rightFast0, long rightFast1, long rightFast2,
+ int rightScale) {
+
+ if (leftSignum == 0) {
+ return (rightSignum == 0);
+ }
+ if (leftSignum != rightSignum) {
+ return false;
+ }
+ if (leftScale != rightScale) {
+ // We know they are not equal because the one with the larger scale has non-zero digits
+ // below the other's scale (since the scale does not include trailing zeroes).
+ return false;
+ }
+ return (
+ leftFast0 == rightFast0 && leftFast1 == rightFast1 && leftFast2 == rightFast2);
+ }
+
+ private static int doCalculateNewFasterHashCode(
+ int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) {
+
+ long longHashCode;
+
+ long key = fast0;
+
+ // Hash code logic from original calculateLongHashCode
+
+ key = (~key) + (key << 21); // key = (key << 21) - key - 1;
+ key = key ^ (key >>> 24);
+ key = (key + (key << 3)) + (key << 8); // key * 265
+ key = key ^ (key >>> 14);
+ key = (key + (key << 2)) + (key << 4); // key * 21
+ key = key ^ (key >>> 28);
+ key = key + (key << 31);
+
+ longHashCode = key;
+
+ key = fast1;
+
+ key = (~key) + (key << 21); // key = (key << 21) - key - 1;
+ key = key ^ (key >>> 24);
+ key = (key + (key << 3)) + (key << 8); // key * 265
+ key = key ^ (key >>> 14);
+ key = (key + (key << 2)) + (key << 4); // key * 21
+ key = key ^ (key >>> 28);
+ key = key + (key << 31);
+
+ longHashCode ^= key;
+
+ key = fast2;
+
+ key = (~key) + (key << 21); // key = (key << 21) - key - 1;
+ key = key ^ (key >>> 24);
+ key = (key + (key << 3)) + (key << 8); // key * 265
+ key = key ^ (key >>> 14);
+ key = (key + (key << 2)) + (key << 4); // key * 21
+ key = key ^ (key >>> 28);
+ key = key + (key << 31);
+
+ longHashCode ^= key;
+
+ key = fastSignum;
+
+ key = (~key) + (key << 21); // key = (key << 21) - key - 1;
+ key = key ^ (key >>> 24);
+ key = (key + (key << 3)) + (key << 8); // key * 265
+ key = key ^ (key >>> 14);
+ key = (key + (key << 2)) + (key << 4); // key * 21
+ key = key ^ (key >>> 28);
+ key = key + (key << 31);
+
+ longHashCode ^= key;
+
+ key = fastIntegerDigitCount;
+
+ key = (~key) + (key << 21); // key = (key << 21) - key - 1;
+ key = key ^ (key >>> 24);
+ key = (key + (key << 3)) + (key << 8); // key * 265
+ key = key ^ (key >>> 14);
+ key = (key + (key << 2)) + (key << 4); // key * 21
+ key = key ^ (key >>> 28);
+ key = key + (key << 31);
+
+ longHashCode ^= key;
+
+ key = fastScale;
+
+ key = (~key) + (key << 21); // key = (key << 21) - key - 1;
+ key = key ^ (key >>> 24);
+ key = (key + (key << 3)) + (key << 8); // key * 265
+ key = key ^ (key >>> 14);
+ key = (key + (key << 2)) + (key << 4); // key * 21
+ key = key ^ (key >>> 28);
+ key = key + (key << 31);
+
+ longHashCode ^= key;
+
+ return (int) longHashCode;
+ }
+
+ private static final int ZERO_NEW_FASTER_HASH_CODE = doCalculateNewFasterHashCode(0, 0, 0, 0, 0, 0);
+
+ /**
+ * Hash code based on (new) decimal representation.
+ *
+ * Faster than fastHashCode().
+ *
+ * Used by map join and other Hive internal purposes where performance is important.
+ *
+ * IMPORTANT: See comments for fastHashCode(), too.
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @return the hash code
+ */
+ public static int fastNewFasterHashCode(
+ int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) {
+ if (fastSignum == 0) {
+ return ZERO_NEW_FASTER_HASH_CODE;
+ }
+ int hashCode = doCalculateNewFasterHashCode(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ return hashCode;
+ }
+
+ /**
+ * This is the original hash code as returned by OldHiveDecimal.
+ *
+ * We need this when the OldHiveDecimal hash code has been exposed and and written or affected
+ * how data is written.
+ *
+ * This method supports compatibility.
+ *
+ * Examples: bucketing and the Hive hash() function.
+ *
+ * NOTE: It is necessary to create a BigDecimal object and use its hash code, so this method is
+ * slow.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @return the hash code
+ */
+ public static int fastHashCode(
+ int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) {
+
+ // OldHiveDecimal returns the hash code of its internal BigDecimal. Our TestHiveDecimal
+ // verifies the OldHiveDecimal.bigDecimalValue() matches (new) HiveDecimal.bigDecimalValue().
+
+ BigDecimal bigDecimal =
+ fastBigDecimalValue(
+ fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+
+ return bigDecimal.hashCode();
+ }
+
+ //************************************************************************************************
+ // Decimal Math.
+
+ public static boolean fastScaleByPowerOfTen(
+ FastHiveDecimal fastDec,
+ int power,
+ FastHiveDecimal fastResult) {
+ return fastScaleByPowerOfTen(
+ fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2,
+ fastDec.fastIntegerDigitCount, fastDec.fastScale,
+ power,
+ fastResult);
+ }
+
+ // NOTE: power can be positive or negative.
+ // NOTE: e.g. power = 2 is effectively multiply by 10^2
+ // NOTE: and power = -3 is multiply by 10^-3
+ public static boolean fastScaleByPowerOfTen(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int power,
+ FastHiveDecimal fastResult) {
+
+ if (fastSignum == 0) {
+ fastResult.fastReset();
+ return true;
+ }
+ if (power == 0) {
+ fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ /*
+ if (!fastResult.fastIsValid()) {
+ fastResult.fastRaiseInvalidException();
+ }
+ */
+ return true;
+ }
+
+ final int absPower = Math.abs(power);
+
+ if (power > 0) {
+
+ int integerRoom;
+ int fractionalRoom;
+ if (fastIntegerDigitCount > 0) {
+
+ // Is there integer room above?
+
+ integerRoom = HiveDecimal.MAX_PRECISION - fastIntegerDigitCount;
+ if (integerRoom < power) {
+ return false;
+ }
+ fastResult.fastSignum = fastSignum;
+ if (fastScale <= power) {
+
+ // All fractional digits become integer digits.
+ final int scaleUp = power - fastScale;
+ if (scaleUp > 0) {
+ if (!fastScaleUp(
+ fast0, fast1, fast2,
+ scaleUp,
+ fastResult)) {
+ throw new RuntimeException("Unexpected");
+ }
+ } else {
+ fastResult.fast0 = fast0;
+ fastResult.fast1 = fast1;
+ fastResult.fast2 = fast2;
+ }
+ fastResult.fastIntegerDigitCount = fastIntegerDigitCount + fastScale + scaleUp;
+ fastResult.fastScale = 0;
+
+ } else {
+
+ // Only a scale adjustment is needed.
+ fastResult.fastSet(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount + power, fastScale - power);
+ }
+ } else {
+
+ // How much can the fraction be moved up?
+
+ final int rawPrecision = fastRawPrecision(fastSignum, fast0, fast1, fast2);
+ final int zeroesBelowDot = fastScale - rawPrecision;
+
+ // Our limit is max precision integer digits + "leading" zeros below the dot.
+ // E.g. 0.00021 has 3 zeroes below the dot.
+ //
+ if (power > HiveDecimal.MAX_PRECISION + zeroesBelowDot) {
+
+ // Fractional part powered up too high.
+ return false;
+ }
+
+ final int newIntegerDigitCount = Math.max(0, power - zeroesBelowDot);
+ if (newIntegerDigitCount > rawPrecision) {
+
+ fastResult.fastSignum = fastSignum;
+ final int scaleUp = newIntegerDigitCount - rawPrecision;
+ if (!fastScaleUp(
+ fast0, fast1, fast2,
+ scaleUp,
+ fastResult)) {
+ throw new RuntimeException("Unexpected");
+ }
+ fastResult.fastIntegerDigitCount = newIntegerDigitCount;
+ fastResult.fastScale = 0;
+ } else {
+ final int newScale = Math.max(0, fastScale - power);
+ fastResult.fastSet(fastSignum, fast0, fast1, fast2, newIntegerDigitCount, newScale);
+ }
+ }
+
+ } else if (fastScale + absPower <= HiveDecimal.MAX_SCALE) {
+
+ // Negative power with range -- adjust the scale.
+
+ final int newScale = fastScale + absPower;
+ final int newIntegerDigitCount = Math.max(0, fastIntegerDigitCount - absPower);
+
+ final int trailingZeroCount =
+ fastTrailingDecimalZeroCount(
+ fast0, fast1, fast2,
+ newIntegerDigitCount, newScale);
+ if (trailingZeroCount > 0) {
+ fastResult.fastSignum = fastSignum;
+ doFastScaleDown(
+ fast0, fast1, fast2,
+ trailingZeroCount, fastResult);
+ fastResult.fastScale = newScale - trailingZeroCount;
+ fastResult.fastIntegerDigitCount = newIntegerDigitCount;
+ } else {
+ fastResult.fastSet(fastSignum, fast0, fast1, fast2,
+ newIntegerDigitCount, newScale);
+ }
+ } else {
+
+ // fastScale + absPower > HiveDecimal.MAX_SCALE
+
+ // Look at getting rid of fractional digits that will now be below HiveDecimal.MAX_SCALE.
+
+ final int scaleDown = fastScale + absPower - HiveDecimal.MAX_SCALE;
+
+ if (scaleDown < HiveDecimal.MAX_SCALE) {
+ if (!fastRoundFractionalHalfUp(
+ fastSignum, fast0, fast1, fast2,
+ scaleDown,
+ fastResult)) {
+ // Overflow.
+ return false;
+ }
+ if (fastResult.fastSignum != 0) {
+
+ fastResult.fastScale = HiveDecimal.MAX_SCALE;
+ fastResult.fastIntegerDigitCount =
+ Math.max(0, fastRawPrecision(fastResult) - fastResult.fastScale);
+
+ final int trailingZeroCount =
+ fastTrailingDecimalZeroCount(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ fastResult.fastIntegerDigitCount, fastResult.fastScale);
+ if (trailingZeroCount > 0) {
+ doFastScaleDown(
+ fastResult,
+ trailingZeroCount,
+ fastResult);
+ fastResult.fastScale -= trailingZeroCount;
+ }
+ }
+ } else {
+ // All precision has been lost -- result is 0.
+ fastResult.fastReset();
+ }
+ }
+
+ /*
+ if (!fastResult.fastIsValid()) {
+ fastResult.fastRaiseInvalidException();
+ }
+ */
+
+ return true;
+ }
+
+ //************************************************************************************************
+ // Decimal Rounding.
+
+ public static boolean doFastRound(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int roundPower,
+ int roundingMode,
+ FastHiveDecimal fastResult) {
+
+ if (fastSignum == 0) {
+
+ // Zero result.
+ fastResult.fastReset();
+ return true;
+ } else if (fastScale == roundPower) {
+
+ // The roundPower same as scale means all zeroes below round point.
+
+ fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ /*
+ if (!fastResult.fastIsValid()) {
+ fastResult.fastRaiseInvalidException();
+ }
+ */
+ return true;
+ }
+
+ if (roundPower > fastScale) {
+
+ // We pretend to add trailing zeroes, EVEN WHEN it would exceed the HiveDecimal.MAX_PRECISION.
+
+ // Copy current value; do not change current scale.
+ fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ /*
+ if (!fastResult.fastIsValid()) {
+ fastResult.fastRaiseInvalidException();
+ }
+ */
+ } else if (roundPower < 0) {
+
+ // roundPower < 0
+ //
+ // Negative scale means we start rounding integer digits.
+ //
+ // The result will integer result will have at least abs(roundPower) trailing digits.
+ //
+ // Examples where the 'r's show the rounding digits:
+ //
+ // round(12500, -3) = 13000 // BigDecimal.ROUND_HALF_UP
+ // rrr
+ //
+ // Or, ceiling(12400.8302, -2) = 12500 // BigDecimal.ROUND_CEILING
+ // rr rrrr
+ //
+ // Notice that any fractional digits will be gone in the result.
+ //
+ switch (roundingMode) {
+ case BigDecimal.ROUND_DOWN:
+ if (!fastRoundIntegerDown(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ roundPower,
+ fastResult)) {
+ return false;
+ }
+ break;
+ case BigDecimal.ROUND_UP:
+ if (!fastRoundIntegerUp(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ roundPower,
+ fastResult)) {
+ return false;
+ }
+ break;
+ case BigDecimal.ROUND_FLOOR:
+ // Round towards negative infinity.
+ if (fastSignum == 1) {
+ if (!fastRoundIntegerDown(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ roundPower,
+ fastResult)) {
+ return false;
+ }
+ } else {
+ if (!fastRoundIntegerUp(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ roundPower,
+ fastResult)) {
+ return false;
+ }
+ if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) {
+ return false;
+ }
+ }
+ break;
+ case BigDecimal.ROUND_CEILING:
+ // Round towards positive infinity.
+ if (fastSignum == 1) {
+ if (!fastRoundIntegerUp(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ roundPower,
+ fastResult)) {
+ return false;
+ }
+ if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) {
+ return false;
+ }
+ } else {
+ if (!fastRoundIntegerDown(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ roundPower,
+ fastResult)) {
+ return false;
+ }
+ }
+ break;
+ case BigDecimal.ROUND_HALF_UP:
+ if (!fastRoundIntegerHalfUp(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ roundPower,
+ fastResult)) {
+ return false;
+ }
+ if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) {
+ return false;
+ }
+ break;
+ case BigDecimal.ROUND_HALF_EVEN:
+ if (!fastRoundIntegerHalfEven(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ roundPower,
+ fastResult)) {
+ return false;
+ }
+ if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) {
+ return false;
+ }
+ break;
+ default:
+ throw new RuntimeException("Unsupported rounding mode " + roundingMode);
+ }
+
+ // The fastRoundInteger* methods remove all fractional digits, set fastIntegerDigitCount, and
+ // set fastScale to 0.
+ return true;
+
+ } else {
+
+ // roundPower < fastScale
+
+ // Do rounding of fractional digits.
+ final int scaleDown = fastScale - roundPower;
+ switch (roundingMode) {
+ case BigDecimal.ROUND_DOWN:
+ fastRoundFractionalDown(
+ fastSignum, fast0, fast1, fast2,
+ scaleDown,
+ fastResult);
+ break;
+ case BigDecimal.ROUND_UP:
+ if (!fastRoundFractionalUp(
+ fastSignum, fast0, fast1, fast2,
+ scaleDown,
+ fastResult)) {
+ return false;
+ }
+ break;
+ case BigDecimal.ROUND_FLOOR:
+ // Round towards negative infinity.
+ if (fastSignum == 1) {
+ fastRoundFractionalDown(
+ fastSignum, fast0, fast1, fast2,
+ scaleDown,
+ fastResult);
+ } else {
+ if (!fastRoundFractionalUp(
+ fastSignum, fast0, fast1, fast2,
+ scaleDown,
+ fastResult)) {
+ return false;
+ }
+ }
+ break;
+ case BigDecimal.ROUND_CEILING:
+ // Round towards positive infinity.
+ if (fastSignum == 1) {
+ if (!fastRoundFractionalUp(
+ fastSignum, fast0, fast1, fast2,
+ scaleDown,
+ fastResult)) {
+ return false;
+ }
+ } else {
+ fastRoundFractionalDown(
+ fastSignum, fast0, fast1, fast2,
+ scaleDown,
+ fastResult);
+ }
+ break;
+ case BigDecimal.ROUND_HALF_UP:
+ if (!fastRoundFractionalHalfUp(
+ fastSignum, fast0, fast1, fast2,
+ scaleDown,
+ fastResult)) {
+ return false;
+ }
+ break;
+ case BigDecimal.ROUND_HALF_EVEN:
+ if (!fastRoundFractionalHalfEven(
+ fastSignum, fast0, fast1, fast2,
+ scaleDown,
+ fastResult)) {
+ return false;
+ }
+ break;
+ default:
+ throw new RuntimeException("Unsupported rounding mode " + roundingMode);
+ }
+ if (fastResult.fastSignum == 0) {
+ fastResult.fastScale = 0;
+ /*
+ if (!fastResult.fastIsValid()) {
+ fastResult.fastRaiseInvalidException();
+ }
+ */
+ } else {
+ final int rawPrecision = fastRawPrecision(fastResult);
+ fastResult.fastIntegerDigitCount = Math.max(0, rawPrecision - roundPower);
+ fastResult.fastScale = roundPower;
+
+ // Trim trailing zeroes and re-adjust scale.
+ final int trailingZeroCount =
+ fastTrailingDecimalZeroCount(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ fastResult.fastIntegerDigitCount, fastResult.fastScale);
+ if (trailingZeroCount > 0) {
+ doFastScaleDown(
+ fastResult,
+ trailingZeroCount,
+ fastResult);
+ fastResult.fastScale -= trailingZeroCount;
+ /*
+ if (!fastResult.fastIsValid()) {
+ fastResult.fastRaiseInvalidException();
+ }
+ */
+ }
+ }
+ }
+
+ return true;
+ }
+
+ public static boolean fastRound(
+ FastHiveDecimal fastDec,
+ int newScale, int roundingMode,
+ FastHiveDecimal fastResult) {
+ return fastRound(
+ fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2,
+ fastDec.fastIntegerDigitCount, fastDec.fastScale,
+ newScale, roundingMode,
+ fastResult);
+ }
+
+ public static boolean fastRound(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int newScale, int roundingMode,
+ FastHiveDecimal fastResult) {
+ return doFastRound(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ newScale, roundingMode,
+ fastResult);
+ }
+
+ private static boolean isRoundPortionAllZeroes(
+ long fast0, long fast1, long fast2,
+ int roundingPoint) {
+
+ boolean isRoundPortionAllZeroes;
+ if (roundingPoint < LONGWORD_DECIMAL_DIGITS) {
+
+ // Lowest word gets integer rounding.
+
+ // Factor includes scale.
+ final long roundPointFactor = powerOfTenTable[roundingPoint];
+
+ isRoundPortionAllZeroes = (fast0 % roundPointFactor == 0);
+
+ } else if (roundingPoint < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ // Middle word gets integer rounding.
+
+ final int adjustedRoundingPoint = roundingPoint - LONGWORD_DECIMAL_DIGITS;
+
+ if (adjustedRoundingPoint == 0) {
+ isRoundPortionAllZeroes = (fast0 == 0);
+ } else {
+
+ // Factor includes scale.
+ final long roundPointFactor = powerOfTenTable[adjustedRoundingPoint];
+
+ final long roundPortion = fast1 % roundPointFactor;
+ isRoundPortionAllZeroes = (fast0 == 0 && roundPortion == 0);
+ }
+
+ } else {
+
+ // High word gets integer rounding.
+
+ final int adjustedRoundingPoint = roundingPoint - TWO_X_LONGWORD_DECIMAL_DIGITS;
+
+ if (adjustedRoundingPoint == 0) {
+ isRoundPortionAllZeroes = (fast0 == 0 && fast1 == 0);
+ } else {
+
+ // Factor includes scale.
+ final long roundPointFactor = powerOfTenTable[adjustedRoundingPoint];
+
+ final long roundPortion = fast2 % roundPointFactor;
+ isRoundPortionAllZeroes = (fast0 == 0 && fast1 == 0 && roundPortion == 0);
+ }
+ }
+ return isRoundPortionAllZeroes;
+ }
+
+ private static boolean isRoundPortionHalfUp(
+ long fast0, long fast1, long fast2,
+ int roundingPoint) {
+
+ boolean isRoundPortionHalfUp;
+ if (roundingPoint < LONGWORD_DECIMAL_DIGITS) {
+
+ // Lowest word gets integer rounding.
+
+ // Divide down just before round point to get round digit.
+ final long withRoundDigit = fast0 / powerOfTenTable[roundingPoint - 1];
+ final long roundDigit = withRoundDigit % 10;
+
+ isRoundPortionHalfUp = (roundDigit >= 5);
+
+ } else if (roundingPoint < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ // Middle word gets integer rounding.
+
+ final int adjustedRoundingPoint = roundingPoint - LONGWORD_DECIMAL_DIGITS;
+
+ long roundDigit;
+ if (adjustedRoundingPoint == 0) {
+ // Grab round digit from lowest word.
+ roundDigit = fast0 / (MULTIPLER_LONGWORD_DECIMAL / 10);
+ } else {
+ // Divide down just before scaleDown to get round digit.
+ final long withRoundDigit = fast1 / powerOfTenTable[adjustedRoundingPoint - 1];
+ roundDigit = withRoundDigit % 10;
+ }
+
+ isRoundPortionHalfUp = (roundDigit >= 5);
+
+ } else {
+
+ // High word gets integer rounding.
+
+ final int adjustedRoundingPoint = roundingPoint - TWO_X_LONGWORD_DECIMAL_DIGITS;
+
+ long roundDigit;
+ if (adjustedRoundingPoint == 0) {
+ // Grab round digit from middle word.
+ roundDigit = fast1 / (MULTIPLER_LONGWORD_DECIMAL / 10);
+ } else {
+ // Divide down just before scaleDown to get round digit.
+ final long withRoundDigit = fast2 / powerOfTenTable[adjustedRoundingPoint - 1];
+ roundDigit = withRoundDigit % 10;
+ }
+
+ isRoundPortionHalfUp = (roundDigit >= 5);
+
+ }
+ return isRoundPortionHalfUp;
+ }
+
+ private static boolean isRoundPortionHalfEven(
+ long fast0, long fast1, long fast2,
+ int roundingPoint) {
+
+ boolean isRoundPortionHalfEven;
+ if (roundingPoint < LONGWORD_DECIMAL_DIGITS) {
+
+ // Lowest word gets integer rounding.
+
+ // Divide down just before scaleDown to get round digit.
+ final long roundDivisor = powerOfTenTable[roundingPoint - 1];
+ final long withRoundDigit = fast0 / roundDivisor;
+ final long roundDigit = withRoundDigit % 10;
+ final long fast0Scaled = withRoundDigit / 10;
+
+ if (roundDigit > 5) {
+ isRoundPortionHalfEven = true;
+ } else if (roundDigit == 5) {
+ boolean exactlyOneHalf;
+ if (roundingPoint - 1 == 0) {
+ // Fraction below 0.5 is implicitly 0.
+ exactlyOneHalf = true;
+ } else {
+ exactlyOneHalf = (fast0 % roundDivisor == 0);
+ }
+
+ // When fraction is exactly 0.5 and lowest new digit is odd, go towards even.
+ if (exactlyOneHalf) {
+ isRoundPortionHalfEven = (fast0Scaled % 2 == 1);
+ } else {
+ isRoundPortionHalfEven = true;
+ }
+ } else {
+ isRoundPortionHalfEven = false;
+ }
+
+ } else if (roundingPoint < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ // Middle word gets integer rounding.
+
+ final int adjustedRoundingPoint = roundingPoint - LONGWORD_DECIMAL_DIGITS;
+
+ long roundDigit;
+ long fast1Scaled;
+ if (adjustedRoundingPoint == 0) {
+ // Grab round digit from lowest word.
+ final long roundDivisor = MULTIPLER_LONGWORD_DECIMAL / 10;
+ roundDigit = fast0 / roundDivisor;
+ fast1Scaled = fast1;
+ if (roundDigit > 5) {
+ isRoundPortionHalfEven = true;
+ } else if (roundDigit == 5) {
+ boolean exactlyOneHalf = (fast0 % roundDivisor == 0);
+
+ // When fraction is exactly 0.5 and lowest new digit is odd, go towards even.
+ if (exactlyOneHalf) {
+ isRoundPortionHalfEven = (fast1Scaled % 2 == 1);
+ } else {
+ isRoundPortionHalfEven = true;
+ }
+ } else {
+ isRoundPortionHalfEven = false;
+ }
+ } else {
+ // Divide down just before scaleDown to get round digit.
+ final long roundDivisor = powerOfTenTable[adjustedRoundingPoint - 1];
+ final long withRoundDigit = fast1 / roundDivisor;
+ roundDigit = withRoundDigit % 10;
+ fast1Scaled = withRoundDigit / 10;
+ if (roundDigit > 5) {
+ isRoundPortionHalfEven = true;
+ } else if (roundDigit == 5) {
+ boolean exactlyOneHalf;
+ if (adjustedRoundingPoint - 1 == 0) {
+ // Just examine the lower word.
+ exactlyOneHalf = (fast0 == 0);
+ } else {
+ exactlyOneHalf = (fast0 == 0 && fast1 % roundDivisor == 0);
+ }
+
+ // When fraction is exactly 0.5 and lowest new digit is odd, go towards even.
+ if (exactlyOneHalf) {
+ isRoundPortionHalfEven = (fast1Scaled % 2 == 1);
+ } else {
+ isRoundPortionHalfEven = true;
+ }
+ } else {
+ isRoundPortionHalfEven = false;
+ }
+ }
+
+ } else {
+
+ // High word gets integer rounding.
+
+ final int adjustedRoundingPoint = roundingPoint - TWO_X_LONGWORD_DECIMAL_DIGITS;
+
+ long roundDigit;
+ long fast2Scaled;
+ if (adjustedRoundingPoint == 0) {
+ // Grab round digit from middle word.
+ final long roundDivisor = MULTIPLER_LONGWORD_DECIMAL / 10;
+ roundDigit = fast1 / roundDivisor;
+ fast2Scaled = fast2;
+ if (roundDigit > 5) {
+ isRoundPortionHalfEven = true;
+ } else if (roundDigit == 5) {
+ boolean exactlyOneHalf = (fast1 % roundDivisor == 0 && fast0 == 0);
+
+ // When fraction is exactly 0.5 and lowest new digit is odd, go towards even.
+ if (exactlyOneHalf) {
+ isRoundPortionHalfEven = (fast2Scaled % 2 == 1);
+ } else {
+ isRoundPortionHalfEven = true;
+ }
+ } else {
+ isRoundPortionHalfEven = false;
+ }
+ } else {
+ // Divide down just before scaleDown to get round digit.
+ final long roundDivisor = powerOfTenTable[adjustedRoundingPoint - 1];
+ final long withRoundDigit = fast2 / roundDivisor;
+ roundDigit = withRoundDigit % 10;
+ fast2Scaled = withRoundDigit / 10;
+ if (roundDigit > 5) {
+ isRoundPortionHalfEven = true;
+ } else if (roundDigit == 5) {
+ boolean exactlyOneHalf;
+ if (adjustedRoundingPoint - 1 == 0) {
+ // Just examine the middle and lower words.
+ exactlyOneHalf = (fast1 == 0 && fast0 == 0);
+ } else {
+ exactlyOneHalf = (fast2 % roundDivisor == 0 && fast1 == 0 && fast0 == 0);
+ }
+
+ // When fraction is exactly 0.5 and lowest new digit is odd, go towards even.
+ if (exactlyOneHalf) {
+ isRoundPortionHalfEven = (fast2Scaled % 2 == 1);
+ } else {
+ isRoundPortionHalfEven = true;
+ }
+ } else {
+ isRoundPortionHalfEven = false;
+ }
+ }
+ }
+ return isRoundPortionHalfEven;
+ }
+
+ private static void doClearRoundIntegerPortionAndAddOne(
+ long fast0, long fast1, long fast2,
+ int absRoundPower,
+ FastHiveDecimal fastResult) {
+
+ long result0;
+ long result1;
+ long result2;
+
+ if (absRoundPower < LONGWORD_DECIMAL_DIGITS) {
+
+ // Lowest word gets integer rounding.
+
+ // Clear rounding portion in lower longword and add 1 at right scale (roundMultiplyFactor).
+
+ final long roundFactor = powerOfTenTable[absRoundPower];
+
+ final long r0 =
+ ((fast0 / roundFactor) * roundFactor)
+ + roundFactor;
+ result0 = r0 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r1 =
+ fast1
+ + r0 / MULTIPLER_LONGWORD_DECIMAL;
+ result1 = r1 % MULTIPLER_LONGWORD_DECIMAL;
+ result2 =
+ fast2
+ + r1 / MULTIPLER_LONGWORD_DECIMAL;
+
+ } else if (absRoundPower < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ // Middle word gets integer rounding; lower longword is cleared.
+
+ final int adjustedAbsPower = absRoundPower - LONGWORD_DECIMAL_DIGITS;
+
+ // Clear rounding portion in middle longword and add 1 at right scale (roundMultiplyFactor);
+ // lower longword result is 0;
+
+ final long roundFactor = powerOfTenTable[adjustedAbsPower];
+
+ result0 = 0;
+ final long r1 =
+ ((fast1 / roundFactor) * roundFactor)
+ + roundFactor;
+ result1 = r1 % MULTIPLER_LONGWORD_DECIMAL;
+ result2 =
+ fast2
+ + r1 / MULTIPLER_LONGWORD_DECIMAL;
+
+ } else {
+
+ // High word gets integer rounding; middle and lower longwords are cleared.
+
+ final int adjustedAbsPower = absRoundPower - TWO_X_LONGWORD_DECIMAL_DIGITS;
+
+ // Clear rounding portion in high longword and add 1 at right scale (roundMultiplyFactor);
+ // middle and lower longwords result is 0;
+
+ final long roundFactor = powerOfTenTable[adjustedAbsPower];
+
+ result0 = 0;
+ result1 = 0;
+ result2 =
+ ((fast2 / roundFactor) * roundFactor)
+ + roundFactor;
+
+ }
+
+ fastResult.fast0 = result0;
+ fastResult.fast1 = result1;
+ fastResult.fast2 = result2;
+ }
+
+ private static void doClearRoundIntegerPortion(
+ long fast0, long fast1, long fast2,
+ int absRoundPower,
+ FastHiveDecimal fastResult) {
+
+ long result0;
+ long result1;
+ long result2;
+
+ if (absRoundPower < LONGWORD_DECIMAL_DIGITS) {
+
+ // Lowest word gets integer rounding.
+
+ // Clear rounding portion in lower longword and add 1 at right scale (roundMultiplyFactor).
+
+ final long roundFactor = powerOfTenTable[absRoundPower];
+ // final long roundMultiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - absRoundPower];
+
+ result0 =
+ ((fast0 / roundFactor) * roundFactor);
+ result1 = fast1;
+ result2 = fast2;
+
+ } else if (absRoundPower < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ // Middle word gets integer rounding; lower longword is cleared.
+
+ final int adjustedAbsPower = absRoundPower - LONGWORD_DECIMAL_DIGITS;
+
+ // Clear rounding portion in middle longword and add 1 at right scale (roundMultiplyFactor);
+ // lower longword result is 0;
+
+ final long roundFactor = powerOfTenTable[adjustedAbsPower];
+
+ result0 = 0;
+ result1 =
+ ((fast1 / roundFactor) * roundFactor);
+ result2 = fast2;
+
+ } else {
+
+ // High word gets integer rounding; middle and lower longwords are cleared.
+
+ final int adjustedAbsPower = absRoundPower - TWO_X_LONGWORD_DECIMAL_DIGITS;
+
+ // Clear rounding portion in high longword and add 1 at right scale (roundMultiplyFactor);
+ // middle and lower longwords result is 0;
+
+ final long roundFactor = powerOfTenTable[adjustedAbsPower];
+
+ result0 = 0;
+ result1 = 0;
+ result2 =
+ ((fast2 / roundFactor) * roundFactor);
+
+ }
+
+ fastResult.fast0 = result0;
+ fastResult.fast1 = result1;
+ fastResult.fast2 = result2;
+ }
+
+ /**
+ * Fast decimal integer part rounding ROUND_UP.
+ *
+ * ceiling(12400.8302, -2) = 12500 // E.g. Positive case FAST_ROUND_CEILING
+ * rr rrrr
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @param roundPower the power to round to
+ * @param fastResult an object to reuse
+ * @return was the operation successful
+ */
+ public static boolean fastRoundIntegerUp(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int roundPower,
+ FastHiveDecimal fastResult) {
+
+ /*
+ * Basic algorithm:
+ *
+ * 1. Determine if rounding part is non-zero for rounding.
+ * 2. Scale away fractional digits if present.
+ * 3. If rounding, clear integer rounding portion and add 1.
+ *
+ */
+
+ if (roundPower >= 0) {
+ throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")");
+ }
+
+ final int absRoundPower = -roundPower;
+ if (fastIntegerDigitCount < absRoundPower) {
+
+ // Above decimal.
+ return false;
+ }
+
+ final int roundingPoint = absRoundPower + fastScale;
+ if (roundingPoint > HiveDecimal.MAX_PRECISION) {
+
+ // Value becomes null for rounding beyond.
+ return false;
+ }
+
+ // First, determine whether rounding is necessary based on rounding point, which is inside
+ // integer part. And, get rid of any fractional digits. The result scale will be 0.
+ //
+ boolean isRoundPortionAllZeroes =
+ isRoundPortionAllZeroes(
+ fast0, fast1, fast2,
+ roundingPoint);
+
+ // If necessary, divide and multiply to get rid of fractional digits.
+ if (fastScale == 0) {
+ fastResult.fast0 = fast0;
+ fastResult.fast1 = fast1;
+ fastResult.fast2 = fast2;
+ } else {
+ doFastScaleDown(
+ fast0, fast1, fast2,
+ /* scaleDown */ fastScale,
+ fastResult);
+ }
+
+ // The fractional digits are gone; when rounding, clear remaining round digits and add 1.
+ if (!isRoundPortionAllZeroes) {
+
+ doClearRoundIntegerPortionAndAddOne(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ absRoundPower,
+ fastResult);
+ }
+
+ if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) {
+ fastResult.fastSignum = 0;
+ fastResult.fastIntegerDigitCount = 0;
+ fastResult.fastScale = 0;
+ } else {
+ fastResult.fastSignum = fastSignum;
+ fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult);
+ fastResult.fastScale = 0;
+ }
+
+ return true;
+ }
+
+ /**
+ * Fast decimal scale down by factor of 10 with rounding ROUND_DOWN.
+ *
+ * The fraction being scaled away is thrown away.
+ *
+ * The signum will be updated if the result is 0, otherwise the original sign is unchanged.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @param roundPower the power to round to
+ * @param fastResult an object to reuse
+ * @return was the operation successful?
+ */
+ public static boolean fastRoundIntegerDown(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int roundPower,
+ FastHiveDecimal fastResult) {
+
+ /*
+ * Basic algorithm:
+ *
+ * 1. Scale away fractional digits if present.
+ * 2. Clear integer rounding portion.
+ *
+ */
+
+ if (roundPower >= 0) {
+ throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")");
+ }
+
+ final int absRoundPower = -roundPower;
+ if (fastIntegerDigitCount < absRoundPower) {
+
+ // Zero result.
+ fastResult.fastReset();
+ return true;
+ }
+
+ final int roundingPoint = absRoundPower + fastScale;
+ if (roundingPoint > HiveDecimal.MAX_PRECISION) {
+
+ // Value becomes zero for rounding beyond.
+ fastResult.fastReset();
+ return true;
+ }
+
+ // If necessary, divide and multiply to get rid of fractional digits.
+ if (fastScale == 0) {
+ fastResult.fast0 = fast0;
+ fastResult.fast1 = fast1;
+ fastResult.fast2 = fast2;
+ } else {
+ doFastScaleDown(
+ fast0, fast1, fast2,
+ /* scaleDown */ fastScale,
+ fastResult);
+ }
+
+ // The fractional digits are gone; clear remaining round digits.
+ doClearRoundIntegerPortion(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ absRoundPower,
+ fastResult);
+
+ if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) {
+ fastResult.fastIntegerDigitCount = 0;
+ fastResult.fastScale = 0;
+ } else {
+ fastResult.fastSignum = 0;
+ fastResult.fastSignum = fastSignum;
+ fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult);
+ fastResult.fastScale = 0;
+ }
+
+ return true;
+ }
+
+ /**
+ * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP.
+ *
+ * When the fraction being scaled away is >= 0.5, the add 1.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @param roundPower the power to round to
+ * @param fastResult an object to reuse
+ * @return was the operation successful?
+ */
+ public static boolean fastRoundIntegerHalfUp(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int roundPower,
+ FastHiveDecimal fastResult) {
+
+ /*
+ * Basic algorithm:
+ *
+ * 1. Determine if rounding digit is >= 5 for rounding.
+ * 2. Scale away fractional digits if present.
+ * 3. If rounding, clear integer rounding portion and add 1.
+ *
+ */
+
+ if (roundPower >= 0) {
+ throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")");
+ }
+
+ final int absRoundPower = -roundPower;
+ if (fastIntegerDigitCount < absRoundPower) {
+
+ // Zero result.
+ fastResult.fastReset();
+ return true;
+ }
+
+ final int roundingPoint = absRoundPower + fastScale;
+ if (roundingPoint > HiveDecimal.MAX_PRECISION) {
+
+ // Value becomes zero for rounding beyond.
+ fastResult.fastReset();
+ return true;
+ }
+
+ // First, determine whether rounding is necessary based on rounding point, which is inside
+ // integer part. And, get rid of any fractional digits. The result scale will be 0.
+ //
+ boolean isRoundPortionHalfUp =
+ isRoundPortionHalfUp(
+ fast0, fast1, fast2,
+ roundingPoint);
+
+ // If necessary, divide and multiply to get rid of fractional digits.
+ if (fastScale == 0) {
+ fastResult.fast0 = fast0;
+ fastResult.fast1 = fast1;
+ fastResult.fast2 = fast2;
+ } else {
+ doFastScaleDown(
+ fast0, fast1, fast2,
+ /* scaleDown */ fastScale,
+ fastResult);
+ }
+
+ // The fractional digits are gone; when rounding, clear remaining round digits and add 1.
+ if (isRoundPortionHalfUp) {
+
+ doClearRoundIntegerPortionAndAddOne(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ absRoundPower,
+ fastResult);
+ } else {
+
+ doClearRoundIntegerPortion(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ absRoundPower,
+ fastResult);
+ }
+
+ if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) {
+ fastResult.fastSignum = 0;
+ fastResult.fastIntegerDigitCount = 0;
+ fastResult.fastScale = 0;
+ } else {
+ fastResult.fastSignum = fastSignum;
+ fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult);
+ fastResult.fastScale = 0;
+ }
+
+ return true;
+ }
+
+ /**
+ * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_EVEN.
+ *
+ * When the fraction being scaled away is exactly 0.5, then round and add 1 only if aaa.
+ * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1.
+ * Otherwise, throw away fraction.
+ *
+ * The signum will be updated if the result is 0, otherwise the original sign is unchanged.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @param roundPower the power to round to
+ * @param fastResult an object to reuse
+ * @return was the operation successful?
+ */
+ public static boolean fastRoundIntegerHalfEven(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int roundPower,
+ FastHiveDecimal fastResult) {
+
+ /*
+ * Basic algorithm:
+ *
+ * 1. Determine if rounding part meets banker's rounding rules for rounding.
+ * 2. Scale away fractional digits if present.
+ * 3. If rounding, clear integer rounding portion and add 1.
+ *
+ */
+
+ if (roundPower >= 0) {
+ throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")");
+ }
+
+ final int absRoundPower = -roundPower;
+ if (fastIntegerDigitCount < absRoundPower) {
+
+ // Zero result.
+ fastResult.fastReset();
+ }
+
+ final int roundingPoint = absRoundPower + fastScale;
+ if (roundingPoint > HiveDecimal.MAX_PRECISION) {
+
+ // Value becomes zero for rounding beyond.
+ fastResult.fastReset();
+ return true;
+ }
+
+ // First, determine whether rounding is necessary based on rounding point, which is inside
+ // integer part. And, get rid of any fractional digits. The result scale will be 0.
+ //
+ boolean isRoundPortionHalfEven =
+ isRoundPortionHalfEven(
+ fast0, fast1, fast2,
+ roundingPoint);
+
+ // If necessary, divide and multiply to get rid of fractional digits.
+ if (fastScale == 0) {
+ fastResult.fast0 = fast0;
+ fastResult.fast1 = fast1;
+ fastResult.fast2 = fast2;
+ } else {
+ doFastScaleDown(
+ fast0, fast1, fast2,
+ /* scaleDown */ fastScale,
+ fastResult);
+ }
+
+ // The fractional digits are gone; when rounding, clear remaining round digits and add 1.
+ if (isRoundPortionHalfEven) {
+
+ doClearRoundIntegerPortionAndAddOne(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ absRoundPower,
+ fastResult);
+ } else {
+
+ doClearRoundIntegerPortion(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ absRoundPower,
+ fastResult);
+ }
+
+ if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) {
+ fastResult.fastSignum = 0;
+ fastResult.fastIntegerDigitCount = 0;
+ fastResult.fastScale = 0;
+ } else {
+ fastResult.fastSignum = fastSignum;
+ fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult);
+ fastResult.fastScale = 0;
+ }
+
+ return true;
+ }
+
+ /**
+ * Fast decimal scale down by factor of 10 and do not allow rounding.
+ *
+ * When the fraction being scaled away is non-zero, return false.
+ *
+ * The signum will be updated if the result is 0, otherwise the original sign
+ * is unchanged.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the digits to scale down by
+ * @param fastResult an object to reuse
+ * @return was the operation successful?
+ */
+ public static boolean fastScaleDownNoRound(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int scaleDown,
+ FastHiveDecimal fastResult) {
+ if (scaleDown < 1 || scaleDown >= THREE_X_LONGWORD_DECIMAL_DIGITS - 1) {
+ throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 3*16 - 1 (scaleDown " + scaleDown + ")");
+ }
+
+ // Adjust all longs using power 10 division/remainder.
+ long result0;
+ long result1;
+ long result2;
+ if (scaleDown < LONGWORD_DECIMAL_DIGITS) {
+
+ // Part of lowest word survives.
+
+ final long divideFactor = powerOfTenTable[scaleDown];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown];
+
+ final long throwAwayFraction = fast0 % divideFactor;
+
+ if (throwAwayFraction != 0) {
+ return false;
+ }
+ result0 =
+ fast0 / divideFactor
+ + ((fast1 % divideFactor) * multiplyFactor);
+ result1 =
+ fast1 / divideFactor
+ + ((fast2 % divideFactor) * multiplyFactor);
+ result2 =
+ fast2 / divideFactor;
+
+ } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ // Throw away lowest word.
+
+ final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS;
+
+ final long divideFactor = powerOfTenTable[adjustedScaleDown];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown];
+
+ boolean isThrowAwayFractionZero;
+ if (adjustedScaleDown == 0) {
+ isThrowAwayFractionZero = (fast0 == 0);
+ } else {
+ final long throwAwayFraction = fast1 % divideFactor;
+ isThrowAwayFractionZero = (throwAwayFraction == 0 && fast0 == 0);
+ }
+
+ if (!isThrowAwayFractionZero) {
+ return false;
+ }
+ result0 =
+ fast1 / divideFactor
+ + ((fast2 % divideFactor) * multiplyFactor);
+ result1 =
+ fast2 / divideFactor;
+ result2 = 0;
+
+ } else {
+
+ // Throw away middle and lowest words.
+
+ final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS;
+
+ final long divideFactor = powerOfTenTable[adjustedScaleDown];
+
+ boolean isThrowAwayFractionZero;
+ if (adjustedScaleDown == 0) {
+ isThrowAwayFractionZero = (fast0 == 0 && fast1 == 0);
+ } else {
+ final long throwAwayFraction = fast2 % divideFactor;
+ isThrowAwayFractionZero = (throwAwayFraction == 0 && fast0 == 0 && fast1 == 0);
+ }
+
+ if (!isThrowAwayFractionZero) {
+ return false;
+ }
+ result0 =
+ fast2 / divideFactor;
+ result1 = 0;
+ result2 = 0;
+ }
+
+ if (result0 == 0 && result1 == 0 && result2 == 0) {
+ fastResult.fastReset();
+ } else {
+ fastResult.fastSignum = fastSignum;
+ fastResult.fast0 = result0;
+ fastResult.fast1 = result1;
+ fastResult.fast2 = result2;
+ }
+
+ return true;
+ }
+
+ /**
+ * Fast decimal scale down by factor of 10 with rounding ROUND_UP.
+ *
+ * When the fraction being scaled away is non-zero, the add 1.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
+ * @return was the operation successfule?
+ */
+ public static boolean fastRoundFractionalUp(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int scaleDown,
+ FastHiveDecimal fastResult) {
+ if (scaleDown < 1 || scaleDown > HiveDecimal.MAX_SCALE) {
+ throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < " + HiveDecimal.MAX_SCALE + " (scaleDown " + scaleDown + ")");
+ }
+
+ if (scaleDown == HiveDecimal.MAX_SCALE) {
+
+ // Examine all digits being thrown away to determine if result is 0 or 1.
+ if (fast0 == 0 && fast1 == 0 && fast2 == 0) {
+
+ // Zero result.
+ fastResult.fastReset();
+ } else {
+ fastResult.fastSet(fastSignum, /* fast0 */ 1, 0, 0, /* fastIntegerDigitCount */ 1, 0);
+ }
+ return true;
+ }
+
+ boolean isRoundPortionAllZeroes =
+ isRoundPortionAllZeroes(
+ fast0, fast1, fast2,
+ scaleDown);
+
+ doFastScaleDown(
+ fast0, fast1, fast2,
+ scaleDown,
+ fastResult);
+
+ if (!isRoundPortionAllZeroes) {
+ final long r0 = fastResult.fast0 + 1;
+ fastResult.fast0 =
+ r0 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r1 =
+ fastResult.fast1
+ + r0 / MULTIPLER_LONGWORD_DECIMAL;
+ fastResult.fast1 =
+ r1 % MULTIPLER_LONGWORD_DECIMAL;
+ fastResult.fast2 =
+ fastResult.fast2
+ + r1 / MULTIPLER_LONGWORD_DECIMAL;
+ }
+
+ if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) {
+ fastResult.fastSignum = 0;
+ fastResult.fastIntegerDigitCount = 0;
+ fastResult.fastScale = 0;
+ } else {
+ fastResult.fastSignum = fastSignum;
+ }
+
+ return (fastResult.fast2 <= MAX_HIGHWORD_DECIMAL);
+ }
+
+ /**
+ * Fast decimal scale down by factor of 10 with rounding ROUND_DOWN.
+ *
+ * The fraction being scaled away is thrown away.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
+ */
+ public static void fastRoundFractionalDown(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int scaleDown,
+ FastHiveDecimal fastResult) {
+ if (scaleDown < 1 || scaleDown > HiveDecimal.MAX_SCALE) {
+ throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 38 (scaleDown " + scaleDown + ")");
+ }
+
+ if (scaleDown == HiveDecimal.MAX_SCALE) {
+
+ // Complete fractional digits shear off. Zero result.
+ fastResult.fastReset();
+ return;
+ }
+
+ doFastScaleDown(
+ fast0, fast1, fast2,
+ scaleDown,
+ fastResult);
+
+ if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) {
+ fastResult.fastSignum = 0;
+ fastResult.fastIntegerDigitCount = 0;
+ fastResult.fastScale = 0;
+ } else {
+ fastResult.fastSignum = fastSignum;
+ }
+ }
+
+ /**
+ * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP.
+ *
+ * When the fraction being scaled away is >= 0.5, the add 1.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
+ * @return was the operation successfule?
+ */
+ public static boolean fastRoundFractionalHalfUp(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int scaleDown,
+ FastHiveDecimal fastResult) {
+ if (fastSignum == 0) {
+ throw new IllegalArgumentException("Unexpected zero value");
+ }
+ if (scaleDown < 1 || scaleDown > HiveDecimal.MAX_SCALE) {
+ throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 38 (scaleDown " + scaleDown + ")");
+ }
+
+ if (scaleDown == HiveDecimal.MAX_SCALE) {
+
+ // Check highest digit for rounding.
+ final long roundDigit = fast2 / powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1];
+ if (roundDigit < 5) {
+
+ // Zero result.
+ fastResult.fastReset();
+ } else {
+ fastResult.fastSet(fastSignum, /* fast0 */ 1, 0, 0, /* fastIntegerDigitCount */ 1, 0);
+ }
+ return true;
+ }
+
+ boolean isRoundPortionHalfUp =
+ isRoundPortionHalfUp(
+ fast0, fast1, fast2,
+ scaleDown);
+
+ doFastScaleDown(
+ fast0, fast1, fast2,
+ scaleDown,
+ fastResult);
+
+ if (isRoundPortionHalfUp) {
+ final long r0 = fastResult.fast0 + 1;
+ fastResult.fast0 =
+ r0 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r1 =
+ fastResult.fast1
+ + r0 / MULTIPLER_LONGWORD_DECIMAL;
+ fastResult.fast1 =
+ r1 % MULTIPLER_LONGWORD_DECIMAL;
+ fastResult.fast2 =
+ fastResult.fast2
+ + r1 / MULTIPLER_LONGWORD_DECIMAL;
+ }
+
+ if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) {
+ fastResult.fastSignum = 0;
+ fastResult.fastIntegerDigitCount = 0;
+ fastResult.fastScale = 0;
+ } else {
+ fastResult.fastSignum = fastSignum;
+ }
+
+ return (fastResult.fast2 <= MAX_HIGHWORD_DECIMAL);
+ }
+
+ /**
+ * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP.
+ *
+ * When the fraction being scaled away is >= 0.5, the add 1.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fast3 word 3
+ * @param fast4 word 4
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
+ * @return was the operation successfule?
+ */
+ public static boolean fastRoundFractionalHalfUp5Words(
+ int fastSignum, long fast0, long fast1, long fast2, long fast3, long fast4,
+ int scaleDown,
+ FastHiveDecimal fastResult) {
+
+ // Adjust all longs using power 10 division/remainder.
+ long result0;
+ long result1;
+ long result2;
+ long result3;
+ long result4;
+ if (scaleDown < LONGWORD_DECIMAL_DIGITS) {
+
+ // Part of lowest word survives.
+
+ // Divide down just before scaleDown to get round digit.
+ final long withRoundDigit = fast0 / powerOfTenTable[scaleDown - 1];
+ final long roundDigit = withRoundDigit % 10;
+
+ final long divideFactor = powerOfTenTable[scaleDown];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown];
+
+ if (roundDigit < 5) {
+ result0 =
+ withRoundDigit / 10
+ + ((fast1 % divideFactor) * multiplyFactor);
+ result1 =
+ fast1 / divideFactor
+ + ((fast2 % divideFactor) * multiplyFactor);
+ result2 =
+ + fast2 / divideFactor
+ + ((fast3 % divideFactor) * multiplyFactor);
+ result3 =
+ fast3 / divideFactor
+ + ((fast4 % divideFactor) * multiplyFactor);
+ result4 =
+ fast4 / divideFactor;
+ } else {
+ // Add rounding and handle carry.
+ final long r0 =
+ withRoundDigit / 10
+ + ((fast1 % divideFactor) * multiplyFactor)
+ + 1;
+ result0 = r0 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r1 =
+ fast1 / divideFactor
+ + ((fast2 % divideFactor) * multiplyFactor)
+ + r0 / MULTIPLER_LONGWORD_DECIMAL;
+ result1 = r1 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r2 =
+ fast2 / divideFactor +
+ + ((fast3 % divideFactor) * multiplyFactor)
+ + r1 / MULTIPLER_LONGWORD_DECIMAL;
+ result2 = r2 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r3 =
+ fast3 / divideFactor
+ + ((fast4 % divideFactor) * multiplyFactor)
+ + r2 / MULTIPLER_LONGWORD_DECIMAL;
+ result3 = r3 % MULTIPLER_LONGWORD_DECIMAL;
+ result4 =
+ fast4 / divideFactor +
+ r3 % MULTIPLER_LONGWORD_DECIMAL;
+ }
+ } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ // Throw away lowest word.
+
+ final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS;
+
+ long roundDigit;
+ long fast1Scaled;
+ if (adjustedScaleDown == 0) {
+ // Grab round digit from lowest word.
+ roundDigit = fast0 / (MULTIPLER_LONGWORD_DECIMAL / 10);
+ fast1Scaled = fast1;
+ } else {
+ // Divide down just before scaleDown to get round digit.
+ final long withRoundDigit = fast1 / powerOfTenTable[adjustedScaleDown - 1];
+ roundDigit = withRoundDigit % 10;
+ fast1Scaled = withRoundDigit / 10;
+ }
+
+ final long divideFactor = powerOfTenTable[adjustedScaleDown];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown];
+
+ if (roundDigit < 5) {
+ result0 =
+ fast1Scaled
+ + ((fast2 % divideFactor) * multiplyFactor);
+ result1 =
+ fast2 / divideFactor
+ + ((fast3 % divideFactor) * multiplyFactor);
+ result2 =
+ fast3 / divideFactor
+ + ((fast4 % divideFactor) * multiplyFactor);
+ result3 =
+ fast4 / divideFactor;
+ } else {
+ // Add rounding and handle carry.
+ final long r0 =
+ fast1Scaled
+ + ((fast2 % divideFactor) * multiplyFactor)
+ + 1;
+ result0 = r0 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r1 =
+ fast2 / divideFactor
+ + ((fast3 % divideFactor) * multiplyFactor)
+ + r0 / MULTIPLER_LONGWORD_DECIMAL;
+ result1 = r1 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r2 =
+ fast3 / divideFactor
+ + ((fast4 % divideFactor) * multiplyFactor)
+ + r1 / MULTIPLER_LONGWORD_DECIMAL;
+ result2 = r2 % MULTIPLER_LONGWORD_DECIMAL;
+ result3 =
+ fast4 / divideFactor
+ + r2 / MULTIPLER_LONGWORD_DECIMAL;
+ }
+ result4 = 0;
+ } else {
+
+ // Throw away middle and lowest words.
+
+ final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS;
+
+ long roundDigit;
+ long fast2Scaled;
+ if (adjustedScaleDown == 0) {
+ // Grab round digit from middle word.
+ roundDigit = fast1 / (MULTIPLER_LONGWORD_DECIMAL / 10);
+ fast2Scaled = fast2;
+ } else {
+ // Divide down just before scaleDown to get round digit.
+ final long withRoundDigit = fast2 / powerOfTenTable[adjustedScaleDown - 1];
+ roundDigit = withRoundDigit % 10;
+ fast2Scaled = withRoundDigit / 10;
+ }
+
+ final long divideFactor = powerOfTenTable[adjustedScaleDown];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown];
+
+ if (roundDigit < 5) {
+ result0 =
+ fast2Scaled
+ + ((fast3 % divideFactor) * multiplyFactor);
+ result1 =
+ fast3 / divideFactor
+ + ((fast4 % divideFactor) * multiplyFactor);
+ result2 =
+ fast4 / divideFactor;
+ } else {
+ // Add rounding.
+ final long r0 =
+ fast2Scaled
+ + ((fast3 % divideFactor) * multiplyFactor)
+ + 1;
+ result0 = r0 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r1 =
+ fast3 / divideFactor
+ + ((fast4 % divideFactor) * multiplyFactor)
+ + r0 / MULTIPLER_LONGWORD_DECIMAL;
+ result1 = r1 % MULTIPLER_LONGWORD_DECIMAL;
+ result2 =
+ fast4 / divideFactor
+ + r1 / MULTIPLER_LONGWORD_DECIMAL;
+ }
+ result3 = 0;
+ result4 = 0;
+ }
+
+ if (result4 != 0 || result3 != 0) {
+ throw new RuntimeException("Unexpected overflow into result3 or result4");
+ }
+ if (result0 == 0 && result1 == 0 && result2 == 0) {
+ fastResult.fastReset();
+ }
+ fastResult.fastSignum = fastSignum;
+ fastResult.fast0 = result0;
+ fastResult.fast1 = result1;
+ fastResult.fast2 = result2;
+
+ return (result2 <= MAX_HIGHWORD_DECIMAL);
+ }
+
+ /**
+ * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_EVEN.
+ *
+ * When the fraction being scaled away is exactly 0.5, then round and add 1 only if aaa.
+ * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1.
+ * Otherwise, throw away fraction.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
+ * @return was the operation successfule?
+ */
+ public static boolean fastRoundFractionalHalfEven(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int scaleDown,
+ FastHiveDecimal fastResult) {
+ if (scaleDown < 1 || scaleDown > HiveDecimal.MAX_SCALE) {
+ throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 38 (scaleDown " + scaleDown + ")");
+ }
+
+ if (scaleDown == HiveDecimal.MAX_SCALE) {
+
+ // Check for rounding.
+ final long roundDivisor = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1];
+ final long withRoundDigit = fast2 / roundDivisor;
+ final long roundDigit = withRoundDigit % 10;
+ final long fast2Scaled = withRoundDigit / 10;
+ boolean shouldRound;
+ if (roundDigit > 5) {
+ shouldRound = true;
+ } else if (roundDigit == 5) {
+ boolean exactlyOneHalf = (fast2Scaled == 0 && fast1 == 0 && fast0 == 0);
+ if (exactlyOneHalf) {
+ // Round to even 0.
+ shouldRound = false;
+ } else {
+ shouldRound = true;
+ }
+ } else {
+ shouldRound = false;
+ }
+ if (!shouldRound) {
+
+ // Zero result.
+ fastResult.fastReset();
+ } else {
+ fastResult.fastSet(fastSignum, /* fast0 */ 1, 0, 0, /* fastIntegerDigitCount */ 1, 0);
+ }
+ return true;
+ }
+
+ boolean isRoundPortionHalfEven =
+ isRoundPortionHalfEven(
+ fast0, fast1, fast2,
+ scaleDown);
+
+ doFastScaleDown(
+ fast0, fast1, fast2,
+ scaleDown,
+ fastResult);
+
+ if (isRoundPortionHalfEven) {
+ final long r0 = fastResult.fast0 + 1;
+ fastResult.fast0 =
+ r0 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r1 =
+ fastResult.fast1
+ + r0 / MULTIPLER_LONGWORD_DECIMAL;
+ fastResult.fast1 =
+ r1 % MULTIPLER_LONGWORD_DECIMAL;
+ fastResult.fast2 =
+ fastResult.fast2
+ + r1 / MULTIPLER_LONGWORD_DECIMAL;
+ }
+
+ if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) {
+ fastResult.fastSignum = 0;
+ fastResult.fastIntegerDigitCount = 0;
+ fastResult.fastScale = 0;
+ } else {
+ fastResult.fastSignum = fastSignum;
+ }
+
+ return (fastResult.fast2 <= MAX_HIGHWORD_DECIMAL);
+ }
+
+ public static void doFastScaleDown(
+ FastHiveDecimal fastDec,
+ int scaleDown,
+ FastHiveDecimal fastResult) {
+ doFastScaleDown(
+ fastDec.fast0, fastDec.fast1, fastDec.fast2,
+ scaleDown,
+ fastResult);
+ }
+
+ //************************************************************************************************
+ // Decimal Scale Up/Down.
+
+ /**
+ * Fast decimal scale down by factor of 10 with NO rounding.
+ *
+ * The signum will be updated if the result is 0, otherwise the original sign is unchanged.
+ *
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
+ */
+ public static void doFastScaleDown(
+ long fast0, long fast1, long fast2,
+ int scaleDown,
+ FastHiveDecimal fastResult) {
+
+ // Adjust all longs using power 10 division/remainder.
+ long result0;
+ long result1;
+ long result2;
+ if (scaleDown < LONGWORD_DECIMAL_DIGITS) {
+
+ // Part of lowest word survives.
+
+ final long divideFactor = powerOfTenTable[scaleDown];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown];
+
+ result0 =
+ fast0 / divideFactor
+ + ((fast1 % divideFactor) * multiplyFactor);
+ result1 =
+ fast1 / divideFactor
+ + ((fast2 % divideFactor) * multiplyFactor);
+ result2 =
+ fast2 / divideFactor;
+
+ } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ // Throw away lowest word.
+
+ final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS;
+
+ final long divideFactor = powerOfTenTable[adjustedScaleDown];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown];
+
+ result0 =
+ fast1 / divideFactor
+ + ((fast2 % divideFactor) * multiplyFactor);
+ result1 =
+ fast2 / divideFactor;
+ result2 = 0;
+
+ } else {
+
+ // Throw away middle and lowest words.
+
+ final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS;
+
+ result0 =
+ fast2 / powerOfTenTable[adjustedScaleDown];
+ result1 = 0;
+ result2 = 0;
+
+ }
+
+ if (result0 == 0 && result1 == 0 && result2 == 0) {
+ fastResult.fastSignum = 0;
+ }
+ fastResult.fast0 = result0;
+ fastResult.fast1 = result1;
+ fastResult.fast2 = result2;
+ }
+
+ public static boolean fastScaleUp(
+ FastHiveDecimal fastDec,
+ int scaleUp,
+ FastHiveDecimal fastResult) {
+ return
+ fastScaleUp(
+ fastDec.fast0, fastDec.fast1, fastDec.fast2,
+ scaleUp,
+ fastResult);
+ }
+
+ /**
+ * Fast decimal scale up by factor of 10.
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleUp the number of integer digits to scale up by
+ * @param fastResult an object to reuse
+ * @return was the operation successfule?
+ */
+ public static boolean fastScaleUp(
+ long fast0, long fast1, long fast2,
+ int scaleUp,
+ FastHiveDecimal fastResult) {
+ if (scaleUp < 1 || scaleUp >= HiveDecimal.MAX_SCALE) {
+ throw new IllegalArgumentException("Expecting scaleUp > 0 and scaleUp < 38");
+ }
+
+ long result0;
+ long result1;
+ long result2;
+
+ // Each range checks for overflow first, then moves digits.
+ if (scaleUp < HIGHWORD_DECIMAL_DIGITS) {
+ // Need to check if there are overflow digits in the high word.
+
+ final long overflowFactor = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - scaleUp];
+ if (fast2 / overflowFactor != 0) {
+ return false;
+ }
+
+ final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleUp];
+ final long multiplyFactor = powerOfTenTable[scaleUp];
+
+ result2 =
+ fast2 * multiplyFactor
+ + fast1 / divideFactor;
+ result1 =
+ (fast1 % divideFactor) * multiplyFactor
+ + fast0 / divideFactor;
+ result0 =
+ (fast0 % divideFactor) * multiplyFactor;
+ } else if (scaleUp < HIGHWORD_DECIMAL_DIGITS + LONGWORD_DECIMAL_DIGITS) {
+ // High word must be zero. Check for overflow digits in middle word.
+
+ if (fast2 != 0) {
+ return false;
+ }
+
+ final int adjustedScaleUp = scaleUp - HIGHWORD_DECIMAL_DIGITS;
+
+ final int middleDigits = LONGWORD_DECIMAL_DIGITS - adjustedScaleUp;
+ final long overflowFactor = powerOfTenTable[middleDigits];
+ if (fast1 / overflowFactor != 0) {
+ return false;
+ }
+
+ if (middleDigits < HIGHWORD_DECIMAL_DIGITS) {
+ // Must fill high word from both middle and lower longs.
+
+ final int highWordMoreDigits = HIGHWORD_DECIMAL_DIGITS - middleDigits;
+ final long multiplyFactor = powerOfTenTable[highWordMoreDigits];
+
+ final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - highWordMoreDigits];
+
+ result2 =
+ fast1 * multiplyFactor
+ + fast0 / divideFactor;
+ result1 =
+ (fast0 % divideFactor) * multiplyFactor;
+ result0 = 0;
+ } else if (middleDigits == HIGHWORD_DECIMAL_DIGITS) {
+ // Fill high long from middle long, and middle long from lower long.
+
+ result2 = fast1;
+ result1 = fast0;
+ result0 = 0;
+ } else {
+ // Fill high long from some of middle long.
+
+ final int keepMiddleDigits = middleDigits - HIGHWORD_DECIMAL_DIGITS;
+ final long divideFactor = powerOfTenTable[keepMiddleDigits];
+ final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - keepMiddleDigits];
+
+ result2 =
+ fast1 / divideFactor;
+ result1 =
+ (fast1 % divideFactor) * multiplyFactor
+ + fast0 / divideFactor;
+ result0 =
+ (fast0 % divideFactor) * multiplyFactor;
+ }
+ } else {
+ // High and middle word must be zero. Check for overflow digits in lower word.
+
+ if (fast2 != 0 || fast1 != 0) {
+ return false;
+ }
+
+ final int adjustedScaleUp = scaleUp - HIGHWORD_DECIMAL_DIGITS - LONGWORD_DECIMAL_DIGITS;
+
+ final int lowerDigits = LONGWORD_DECIMAL_DIGITS - adjustedScaleUp;
+ final long overflowFactor = powerOfTenTable[lowerDigits];
+ if (fast0 / overflowFactor != 0) {
+ return false;
+ }
+
+ if (lowerDigits < HIGHWORD_DECIMAL_DIGITS) {
+ // Must fill high word from both middle and lower longs.
+
+ final int highWordMoreDigits = HIGHWORD_DECIMAL_DIGITS - lowerDigits;
+ final long multiplyFactor = powerOfTenTable[highWordMoreDigits];
+
+ final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - highWordMoreDigits];
+
+ result2 =
+ fast0 * multiplyFactor;
+ result1 = 0;
+ result0 = 0;
+ } else if (lowerDigits == HIGHWORD_DECIMAL_DIGITS) {
+ // Fill high long from lower long.
+
+ result2 = fast0;
+ result1 = 0;
+ result0 = 0;
+ } else {
+ // Fill high long and middle from some of lower long.
+
+ final int keepLowerDigits = lowerDigits - HIGHWORD_DECIMAL_DIGITS;
+ final long keepLowerDivideFactor = powerOfTenTable[keepLowerDigits];
+ final long keepLowerMultiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - keepLowerDigits];
+
+ result2 =
+ fast0 / keepLowerDivideFactor;
+ result1 =
+ (fast0 % keepLowerDivideFactor) * keepLowerMultiplyFactor;
+ result0 = 0;
+ }
+ }
+
+ if (result0 == 0 && result1 == 0 && result2 == 0) {
+ fastResult.fastSignum = 0;
+ }
+ fastResult.fast0 = result0;
+ fastResult.fast1 = result1;
+ fastResult.fast2 = result2;
+
+ return true;
+ }
+
+ //************************************************************************************************
+ // Decimal Precision / Trailing Zeroes.
+
+ public static int fastLongWordTrailingZeroCount(
+ long longWord) {
+
+ if (longWord == 0) {
+ return LONGWORD_DECIMAL_DIGITS;
+ }
+
+ long factor = 10;
+ for (int i = 0; i < LONGWORD_DECIMAL_DIGITS; i++) {
+ if (longWord % factor != 0) {
+ return i;
+ }
+ factor *= 10;
+ }
+ return 0;
+ }
+
+ public static int fastHighWordTrailingZeroCount(
+ long longWord) {
+
+ if (longWord == 0) {
+ return HIGHWORD_DECIMAL_DIGITS;
+ }
+
+ long factor = 10;
+ for (int i = 0; i < HIGHWORD_DECIMAL_DIGITS; i++) {
+ if (longWord % factor != 0) {
+ return i;
+ }
+ factor *= 10;
+ }
+ return 0;
+ }
+
+ public static int fastLongWordPrecision(
+ long longWord) {
+
+ if (longWord == 0) {
+ return 0;
+ }
+ // Search like binary search to minimize comparisons.
+ if (longWord > 99999999L) {
+ if (longWord > 999999999999L) {
+ if (longWord > 99999999999999L) {
+ if (longWord > 999999999999999L) {
+ return 16;
+ } else {
+ return 15;
+ }
+ } else {
+ if (longWord > 9999999999999L) {
+ return 14;
+ } else {
+ return 13;
+ }
+ }
+ } else {
+ if (longWord > 9999999999L) {
+ if (longWord > 99999999999L) {
+ return 12;
+ } else {
+ return 11;
+ }
+ } else {
+ if (longWord > 999999999L) {
+ return 10;
+ } else {
+ return 9;
+ }
+ }
+ }
+ } else {
+ if (longWord > 9999L) {
+ if (longWord > 999999L) {
+ if (longWord > 9999999L) {
+ return 8;
+ } else {
+ return 7;
+ }
+ } else {
+ if (longWord > 99999L) {
+ return 6;
+ } else {
+ return 5;
+ }
+ }
+ } else {
+ if (longWord > 99L) {
+ if (longWord > 999L) {
+ return 4;
+ } else {
+ return 3;
+ }
+ } else {
+ if (longWord > 9L) {
+ return 2;
+ } else {
+ return 1;
+ }
+ }
+ }
+ }
+ }
+
+ public static int fastHighWordPrecision(
+ long longWord) {
+
+ if (longWord == 0) {
+ return 0;
+ }
+ // 6 highword digits.
+ if (longWord > 999L) {
+ if (longWord > 9999L) {
+ if (longWord > 99999L) {
+ return 6;
+ } else {
+ return 5;
+ }
+ } else {
+ return 4;
+ }
+ } else {
+ if (longWord > 99L) {
+ return 3;
+ } else {
+ if (longWord > 9L) {
+ return 2;
+ } else {
+ return 1;
+ }
+ }
+ }
+ }
+
+ public static int fastSqlPrecision(
+ FastHiveDecimal fastDec) {
+ return
+ fastSqlPrecision(
+ fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2,
+ fastDec.fastIntegerDigitCount, fastDec.fastScale);
+ }
+
+ public static int fastSqlPrecision(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+
+ if (fastSignum == 0) {
+ return 1;
+ }
+
+ final int rawPrecision =
+ fastRawPrecision(fastSignum, fast0, fast1, fast2);
+
+ if (rawPrecision < fastScale) {
+ // This can happen for numbers less than 0.1
+ // For 0.001234: rawPrecision=4, scale=6
+ // In this case, we'll set the type to have the same precision as the scale.
+ return fastScale;
+ }
+ return rawPrecision;
+ }
+
+ public static int fastRawPrecision(
+ FastHiveDecimal fastDec) {
+ return
+ fastRawPrecision(
+ fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2);
+ }
+
+ public static int fastRawPrecision(
+ int fastSignum, long fast0, long fast1, long fast2) {
+
+ if (fastSignum == 0) {
+ return 0;
+ }
+
+ int precision;
+
+ if (fast2 != 0) {
+
+ // 6 highword digits.
+ precision = TWO_X_LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(fast2);
+
+ } else if (fast1 != 0) {
+
+ // Check fast1.
+ precision = LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fast1);
+
+
+ } else {
+
+ // Check fast0.
+ precision = fastLongWordPrecision(fast0);
+
+ }
+
+ return precision;
+ }
+
+ // Determine if all digits below a power is zero.
+ // The lowest digit is power = 0.
+ public static boolean isAllZeroesBelow(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int power) {
+ if (power < 0 || power > HiveDecimal.MAX_SCALE) {
+ throw new IllegalArgumentException("Expecting power >= 0 and power <= 38");
+ }
+
+ if (fastSignum == 0) {
+ return true;
+ }
+
+ if (power >= TWO_X_LONGWORD_DECIMAL_DIGITS) {
+ if (fast0 != 0 || fast1 != 0) {
+ return false;
+ }
+ final int adjustedPower = power - TWO_X_LONGWORD_DECIMAL_DIGITS;
+ if (adjustedPower == 0) {
+ return true;
+ }
+ long remainder = fast2 % powerOfTenTable[adjustedPower];
+ return (remainder == 0);
+ } else if (power >= LONGWORD_DECIMAL_DIGITS) {
+ if (fast0 != 0) {
+ return false;
+ }
+ final int adjustedPower = power - LONGWORD_DECIMAL_DIGITS;
+ if (adjustedPower == 0) {
+ return true;
+ }
+ long remainder = fast1 % powerOfTenTable[adjustedPower];
+ return (remainder == 0);
+ } else {
+ if (power == 0) {
+ return true;
+ }
+ long remainder = fast0 % powerOfTenTable[power];
+ return (remainder == 0);
+ }
+ }
+
+ public static boolean fastExceedsPrecision(
+ long fast0, long fast1, long fast2,
+ int precision) {
+
+ if (precision <= 0) {
+ return true;
+ } else if (precision >= HiveDecimal.MAX_PRECISION) {
+ return false;
+ }
+ final int precisionLessOne = precision - 1;
+
+ // 0 (lowest), 1 (middle), or 2 (high).
+ int wordNum = precisionLessOne / LONGWORD_DECIMAL_DIGITS;
+
+ int digitInWord = precisionLessOne % LONGWORD_DECIMAL_DIGITS;
+
+ final long overLimitInWord = powerOfTenTable[digitInWord + 1] - 1;
+
+ if (wordNum == 0) {
+ if (digitInWord < LONGWORD_DECIMAL_DIGITS - 1) {
+ if (fast0 > overLimitInWord) {
+ return true;
+ }
+ }
+ return (fast1 != 0 || fast2 != 0);
+ } else if (wordNum == 1) {
+ if (digitInWord < LONGWORD_DECIMAL_DIGITS - 1) {
+ if (fast1 > overLimitInWord) {
+ return true;
+ }
+ }
+ return (fast2 != 0);
+ } else {
+ // We've eliminated the highest digit already with HiveDecimal.MAX_PRECISION check above.
+ return (fast2 > overLimitInWord);
+ }
+ }
+
+ public static int fastTrailingDecimalZeroCount(
+ long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+ if (fastScale < 0 || fastScale > HiveDecimal.MAX_SCALE) {
+ throw new IllegalArgumentException("Expecting scale >= 0 and scale <= 38");
+ }
+ if (fastScale == 0) {
+ return 0;
+ }
+
+ final int lowerLongwordDigits = Math.min(fastScale, LONGWORD_DECIMAL_DIGITS);
+ if (lowerLongwordDigits < LONGWORD_DECIMAL_DIGITS || fast0 != 0) {
+ long factor = 10;
+ for (int i = 0; i < lowerLongwordDigits; i++) {
+ if (fast0 % factor != 0) {
+ return i;
+ }
+ factor *= 10;
+ }
+ if (lowerLongwordDigits < LONGWORD_DECIMAL_DIGITS) {
+ return fastScale;
+ }
+ }
+ if (fastScale == LONGWORD_DECIMAL_DIGITS) {
+ return fastScale;
+ }
+ final int middleLongwordDigits = Math.min(fastScale - LONGWORD_DECIMAL_DIGITS, LONGWORD_DECIMAL_DIGITS);
+ if (middleLongwordDigits < LONGWORD_DECIMAL_DIGITS || fast1 != 0) {
+ long factor = 10;
+ for (int i = 0; i < middleLongwordDigits; i++) {
+ if (fast1 % factor != 0) {
+ return LONGWORD_DECIMAL_DIGITS + i;
+ }
+ factor *= 10;
+ }
+ if (middleLongwordDigits < LONGWORD_DECIMAL_DIGITS) {
+ return fastScale;
+ }
+ }
+ if (fastScale == TWO_X_LONGWORD_DECIMAL_DIGITS) {
+ return fastScale;
+ }
+ final int highLongwordDigits = fastScale - TWO_X_LONGWORD_DECIMAL_DIGITS;
+ if (highLongwordDigits < HIGHWORD_DECIMAL_DIGITS || fast2 != 0) {
+ long factor = 10;
+ for (int i = 0; i < highLongwordDigits; i++) {
+ if (fast2 % factor != 0) {
+ return TWO_X_LONGWORD_DECIMAL_DIGITS + i;
+ }
+ factor *= 10;
+ }
+ }
+ return fastScale;
+ }
+
+ public static FastCheckPrecisionScaleStatus fastCheckPrecisionScale(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int maxPrecision, int maxScale) {
+
+ if (fastSignum == 0) {
+ return FastCheckPrecisionScaleStatus.NO_CHANGE;
+ }
+ final int maxIntegerDigitCount = maxPrecision - maxScale;
+ if (fastIntegerDigitCount > maxIntegerDigitCount) {
+ return FastCheckPrecisionScaleStatus.OVERFLOW;
+ }
+ if (fastScale > maxScale) {
+ return FastCheckPrecisionScaleStatus.UPDATE_SCALE_DOWN;
+ }
+ return FastCheckPrecisionScaleStatus.NO_CHANGE;
+ }
+
+ public static boolean fastUpdatePrecisionScale(
+ final int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int maxPrecision, int maxScale, FastCheckPrecisionScaleStatus status,
+ FastHiveDecimal fastResult) {
+
+ switch (status) {
+ case UPDATE_SCALE_DOWN:
+ {
+ fastResult.fastSignum = fastSignum;
+
+ // Throw away lower digits.
+ if (!fastRoundFractionalHalfUp(
+ fastSignum, fast0, fast1, fast2,
+ fastScale - maxScale,
+ fastResult)) {
+ return false;
+ }
+
+ fastResult.fastScale = maxScale;
+
+ // CONSIDER: For now, recompute integerDigitCount...
+ fastResult.fastIntegerDigitCount =
+ Math.max(0, fastRawPrecision(fastResult) - fastResult.fastScale);
+
+ // And, round up may cause us to exceed our precision/scale...
+ final int maxIntegerDigitCount = maxPrecision - maxScale;
+ if (fastResult.fastIntegerDigitCount > maxIntegerDigitCount) {
+ return false;
+ }
+
+ // Scaling down may have opened up trailing zeroes...
+ final int trailingZeroCount =
+ fastTrailingDecimalZeroCount(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ fastResult.fastIntegerDigitCount, fastResult.fastScale);
+ if (trailingZeroCount > 0) {
+ // Scale down again.
+ doFastScaleDown(
+ fastResult,
+ trailingZeroCount,
+ fastResult);
+ fastResult.fastScale -= trailingZeroCount;
+ }
+ }
+ break;
+ default:
+ throw new RuntimeException("Unexpected fast check precision scale status " + status);
+ }
+
+ return true;
+ }
+
+ //************************************************************************************************
+ // Decimal Addition / Subtraction.
+
+ public static boolean doAddSameScaleSameSign(
+ FastHiveDecimal fastLeft,
+ FastHiveDecimal fastRight,
+ FastHiveDecimal fastResult) {
+ return
+ doAddSameScaleSameSign(
+ /* resultSignum */ fastLeft.fastSignum,
+ fastLeft.fast0, fastLeft.fast1, fastLeft.fast2,
+ fastRight.fast0, fastRight.fast1, fastRight.fast2,
+ fastResult);
+ }
+
+ public static boolean doAddSameScaleSameSign(
+ int resultSignum,
+ long left0, long left1, long left2,
+ long right0, long right1, long right2,
+ FastHiveDecimal fastResult) {
+
+ long result0;
+ long result1;
+ long result2;
+
+ final long r0 = left0 + right0;
+ result0 =
+ r0 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r1 =
+ left1
+ + right1
+ + r0 / MULTIPLER_LONGWORD_DECIMAL;
+ result1 =
+ r1 % MULTIPLER_LONGWORD_DECIMAL;
+ result2 =
+ left2
+ + right2
+ + r1 / MULTIPLER_LONGWORD_DECIMAL;
+
+ if (result0 == 0 && result1 == 0 && result2 == 0) {
+ fastResult.fastReset();
+ } else {
+ fastResult.fastSignum = resultSignum;
+ fastResult.fast0 = result0;
+ fastResult.fast1 = result1;
+ fastResult.fast2 = result2;
+ }
+
+ return (result2 <= MAX_HIGHWORD_DECIMAL);
+ }
+
+ public static boolean doSubtractSameScaleNoUnderflow(
+ int resultSignum,
+ FastHiveDecimal fastLeft,
+ FastHiveDecimal fastRight,
+ FastHiveDecimal fastResult) {
+ return
+ doSubtractSameScaleNoUnderflow(
+ resultSignum,
+ fastLeft.fast0, fastLeft.fast1, fastLeft.fast2,
+ fastRight.fast0, fastRight.fast1, fastRight.fast2,
+ fastResult);
+ }
+
+ public static boolean doSubtractSameScaleNoUnderflow(
+ int resultSignum,
+ long left0, long left1, long left2,
+ long right0, long right1, long right2,
+ FastHiveDecimal fastResult) {
+
+ long result0;
+ long result1;
+ long result2;
+
+ final long r0 = left0 - right0;
+ long r1;
+ if (r0 < 0) {
+ result0 = r0 + MULTIPLER_LONGWORD_DECIMAL;
+ r1 = left1 - right1 - 1;
+ } else {
+ result0 = r0;
+ r1 = left1 - right1;
+ }
+ if (r1 < 0) {
+ result1 = r1 + MULTIPLER_LONGWORD_DECIMAL;
+ result2 = left2 - right2 - 1;
+ } else {
+ result1 = r1;
+ result2 = left2 - right2;
+ }
+ if (result2 < 0) {
+ return false;
+ }
+
+ if (result0 == 0 && result1 == 0 && result2 == 0) {
+ fastResult.fastReset();
+ } else {
+ fastResult.fastSignum = resultSignum;
+ fastResult.fast0 = result0;
+ fastResult.fast1 = result1;
+ fastResult.fast2 = result2;
+ }
+
+ return true;
+ }
+
+ public static boolean doSubtractSameScaleNoUnderflow(
+ long left0, long left1, long left2,
+ long right0, long right1, long right2,
+ long[] result) {
+
+ long result0;
+ long result1;
+ long result2;
+
+ final long r0 = left0 - right0;
+ long r1;
+ if (r0 < 0) {
+ result0 = r0 + MULTIPLER_LONGWORD_DECIMAL;
+ r1 = left1 - right1 - 1;
+ } else {
+ result0 = r0;
+ r1 = left1 - right1;
+ }
+ if (r1 < 0) {
+ result1 = r1 + MULTIPLER_LONGWORD_DECIMAL;
+ result2 = left2 - right2 - 1;
+ } else {
+ result1 = r1;
+ result2 = left2 - right2;
+ }
+ if (result2 < 0) {
+ return false;
+ }
+
+ result[0] = result0;
+ result[1] = result1;
+ result[2] = result2;
+
+ return true;
+ }
+
+ private static boolean doAddSameScale(
+ int leftSignum, long leftFast0, long leftFast1, long leftFast2,
+ int rightSignum, long rightFast0, long rightFast1, long rightFast2,
+ int scale,
+ FastHiveDecimal fastResult) {
+
+ if (leftSignum == rightSignum) {
+ if (!doAddSameScaleSameSign(
+ /* resultSignum */ leftSignum,
+ leftFast0, leftFast1, leftFast2,
+ rightFast0, rightFast1, rightFast2,
+ fastResult)) {
+ // Handle overflow precision issue.
+ if (scale > 0) {
+ if (!fastRoundFractionalHalfUp(
+ fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ 1,
+ fastResult)) {
+ return false;
+ }
+ scale--;
+ } else {
+ // Overflow.
+ return false;
+ }
+ }
+ fastResult.fastScale = scale;
+ } else {
+ // Just compare the magnitudes (i.e. signums set to 1).
+ int compareTo =
+ fastCompareTo(
+ 1,
+ leftFast0, leftFast1, leftFast2, 0,
+ 1,
+ rightFast0, rightFast1, rightFast2, 0);
+ if (compareTo == 0) {
+ // They cancel each other.
+ fastResult.fastReset();
+ return true;
+ }
+
+ if (compareTo == 1) {
+ if (!doSubtractSameScaleNoUnderflow(
+ /* resultSignum */ leftSignum,
+ leftFast0, leftFast1, leftFast2,
+ rightFast0, rightFast1, rightFast2,
+ fastResult)) {
+ throw new RuntimeException("Unexpected underflow");
+ }
+ } else {
+ if (!doSubtractSameScaleNoUnderflow(
+ /* resultSignum */ rightSignum,
+ rightFast0, rightFast1, rightFast2,
+ leftFast0, leftFast1, leftFast2,
+ fastResult)) {
+ throw new RuntimeException("Unexpected underflow");
+ }
+ }
+ fastResult.fastScale = scale;
+ }
+
+ if (fastResult.fastSignum != 0) {
+ final int precision = fastRawPrecision(fastResult);
+ fastResult.fastIntegerDigitCount = Math.max(0, precision - fastResult.fastScale);
+ }
+
+ final int resultTrailingZeroCount =
+ fastTrailingDecimalZeroCount(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ fastResult.fastIntegerDigitCount, fastResult.fastScale);
+ if (resultTrailingZeroCount > 0) {
+ doFastScaleDown(
+ fastResult,
+ resultTrailingZeroCount,
+ fastResult);
+ if (fastResult.fastSignum == 0) {
+ fastResult.fastScale = 0;
+ } else {
+ fastResult.fastScale -= resultTrailingZeroCount;
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Handle the common logic at the end of fastAddDifferentScale / fastSubtractDifferentScale that
+ * takes the 5 intermediate result words and fits them into the 3 longword fast result.
+ *
+ */
+ private static boolean doFinishAddSubtractDifferentScale(
+ long result0, long result1, long result2, long result3, long result4,
+ int resultScale,
+ FastHiveDecimal fastResult) {
+
+ int precision;
+ if (result4 != 0) {
+ precision = FOUR_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result4);
+ } else if (result3 != 0) {
+ precision = THREE_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result3);
+ } else if (result2 != 0) {
+ precision = TWO_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result2);
+ } else if (result1 != 0) {
+ precision = LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result1);
+ } else {
+ precision = fastLongWordPrecision(result0);
+ }
+
+ if (precision > HiveDecimal.MAX_PRECISION){
+ final int scaleDown = precision - HiveDecimal.MAX_PRECISION;
+
+ resultScale -= scaleDown;
+ if (resultScale < 0) {
+ // No room.
+ return false;
+ }
+ if (!fastRoundFractionalHalfUp5Words(
+ 1, result0, result1, result2, result3, result4,
+ scaleDown,
+ fastResult)) {
+ // Handle overflow precision issue.
+ if (resultScale > 0) {
+ if (!fastRoundFractionalHalfUp(
+ fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ 1,
+ fastResult)) {
+ throw new RuntimeException("Unexpected overflow");
+ }
+ if (fastResult.fastSignum == 0) {
+ return true;
+ }
+ resultScale--;
+ } else {
+ return false;
+ }
+ }
+
+ precision = fastRawPrecision(1, fastResult.fast0, fastResult.fast1, fastResult.fast2);
+
+ // Stick back into result variables...
+ result0 = fastResult.fast0;
+ result1 = fastResult.fast1;
+ result2 = fastResult.fast2;
+ }
+
+ // Caller will set signum.
+ fastResult.fastSignum = 1;
+ fastResult.fast0 = result0;
+ fastResult.fast1 = result1;
+ fastResult.fast2 = result2;
+ fastResult.fastIntegerDigitCount = Math.max(0, precision - resultScale);
+ fastResult.fastScale = resultScale;
+
+ final int resultTrailingZeroCount =
+ fastTrailingDecimalZeroCount(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ fastResult.fastIntegerDigitCount, fastResult.fastScale);
+ if (resultTrailingZeroCount > 0) {
+ doFastScaleDown(
+ fastResult,
+ resultTrailingZeroCount,
+ fastResult);
+ if (fastResult.fastSignum == 0) {
+ fastResult.fastScale = 0;
+ } else {
+ fastResult.fastScale -= resultTrailingZeroCount;
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Handle decimal subtraction when the values have different scales.
+ */
+ private static boolean fastSubtractDifferentScale(
+ long leftFast0, long leftFast1, long leftFast2,
+ int leftIntegerDigitCount, int leftScale,
+ long rightFast0, long rightFast1, long rightFast2,
+ int rightIntegerDigitCount, int rightScale,
+ FastHiveDecimal fastResult) {
+
+ int diffScale;
+ int resultScale;
+
+ long result0 = 0;
+ long result1 = 0;
+ long result2 = 0;
+ long result3 = 0;
+ long result4 = 0;
+
+ // Since subtraction is not commutative, we can must subtract in the order passed in.
+ if (leftScale > rightScale) {
+
+ // Since left has a longer digit tail and it doesn't move; we will shift the right digits
+ // as we do our addition into the result.
+
+ diffScale = leftScale - rightScale;
+ resultScale = leftScale;
+
+ if (diffScale < LONGWORD_DECIMAL_DIGITS) {
+
+ final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - diffScale];
+ final long multiplyFactor = powerOfTenTable[diffScale];
+
+ final long r0 =
+ leftFast0
+ - (rightFast0 % divideFactor) * multiplyFactor;
+ long r1;
+ if (r0 < 0) {
+ result0 = r0 + MULTIPLER_LONGWORD_DECIMAL;
+ r1 =
+ leftFast1
+ - rightFast0 / divideFactor
+ - (rightFast1 % divideFactor) * multiplyFactor
+ - 1;
+ } else {
+ result0 = r0;
+ r1 =
+ leftFast1
+ - rightFast0 / divideFactor
+ - (rightFast1 % divideFactor) * multiplyFactor;
+ }
+ long r2;
+ if (r1 < 0) {
+ result1 = r1 + MULTIPLER_LONGWORD_DECIMAL;
+ r2 =
+ leftFast2
+ - rightFast1 / divideFactor
+ - (rightFast2 % divideFactor) * multiplyFactor
+ - 1;
+ } else {
+ result1 = r1;
+ r2 =
+ leftFast2
+ - rightFast1 / divideFactor
+ - (rightFast2 % divideFactor) * multiplyFactor;
+ }
+ long r3;
+ if (r2 < 0) {
+ result2 = r2 + MULTIPLER_LONGWORD_DECIMAL;
+ r3 =
+ -(rightFast2 / divideFactor)
+ - 1;
+ } else {
+ result2 = r2;
+ r3 =
+ -(rightFast2 / divideFactor);
+ }
+ long r4;
+ if (r3 < 0) {
+ result3 = r3 + MULTIPLER_LONGWORD_DECIMAL;
+ r4 = - 1;
+ } else {
+ result3 = r3;
+ r4 = 0;
+ }
+ if (r4 != 0) {
+ throw new RuntimeException("Unexpected underflow");
+ }
+
+ } else if (diffScale == LONGWORD_DECIMAL_DIGITS){
+
+ result0 = leftFast0;
+ final long r1 =
+ leftFast1
+ - rightFast0;
+ long r2;
+ if (r1 < 0) {
+ result1 = r1 + MULTIPLER_LONGWORD_DECIMAL;
+ r2 =
+ leftFast2
+ - rightFast1
+ - 1;
+ } else {
+ result1 = r1;
+ r2 =
+ leftFast2
+ - rightFast1;
+ }
+ long r3;
+ if (r2 < 0) {
+ result2 = r2 + MULTIPLER_LONGWORD_DECIMAL;
+ r3 =
+ -rightFast2
+ - 1;
+ } else {
+ result2 = r2;
+ r3 =
+ -rightFast2;
+ }
+ if (r3 != 0) {
+ throw new RuntimeException("Unexpected underflow");
+ }
+
+
+ } else if (diffScale < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ final long divideFactor = powerOfTenTable[TWO_X_LONGWORD_DECIMAL_DIGITS - diffScale];
+ final long multiplyFactor = powerOfTenTable[diffScale - LONGWORD_DECIMAL_DIGITS];
+
+ result0 = leftFast0;
+ final long r1 =
+ leftFast1
+ -(rightFast0 % divideFactor) * multiplyFactor;
+ long r2;
+ if (r1 < 0) {
+ result1 = r1 + MULTIPLER_LONGWORD_DECIMAL;
+ r2 =
+ leftFast2
+ - rightFast0 / divideFactor
+ - (rightFast1 % divideFactor) * multiplyFactor
+ - 1;
+ } else {
+ result1 = r1;
+ r2 =
+ leftFast2
+ - rightFast0 / divideFactor
+ - (rightFast1 % divideFactor) * multiplyFactor;
+ }
+ long r3;
+ if (r2 < 0) {
+ result2 = r2 + MULTIPLER_LONGWORD_DECIMAL;
+ r3 =
+ - rightFast1 / divideFactor
+ - (rightFast2 % divideFactor) * multiplyFactor
+ - 1;
+ } else {
+ result2 = r2;
+ r3 =
+ - rightFast1 / divideFactor
+ - (rightFast2 % divideFactor) * multiplyFactor;
+ }
+ long r4;
+ if (r3 < 0) {
+ result3 = r3 + MULTIPLER_LONGWORD_DECIMAL;
+ r4 =
+ - rightFast2 / divideFactor
+ - 1;
+ } else {
+ result3 = r3;
+ r4 =
+ - rightFast2 / divideFactor;
+ }
+ long r5;
+ if (r4 < 0) {
+ result4 = r4 + MULTIPLER_LONGWORD_DECIMAL;
+ r5 = - 1;
+ } else {
+ result4 = r4;
+ r5 = 0;
+ }
+ if (r5 != 0) {
+ throw new RuntimeException("Unexpected underflow");
+ }
+
+ } else if (diffScale == TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ result0 = leftFast0;
+ result1 = leftFast1;
+ final long r2 =
+ leftFast2
+ - rightFast0;
+ long r3;
+ if (r2 < 0) {
+ result2 = r2 + MULTIPLER_LONGWORD_DECIMAL;
+ r3 =
+ - rightFast1
+ - 1;
+ } else {
+ result2 = r2;
+ r3 =
+ - rightFast1;
+ }
+ long r4;
+ if (r3 < 0) {
+ result3 = r3 + MULTIPLER_LONGWORD_DECIMAL;
+ r4 =
+ -rightFast2
+ - 1;
+ } else {
+ result3 = r3;
+ r4 =
+ -rightFast2;
+ }
+ long r5;
+ if (r4 < 0) {
+ result4 = r4 + MULTIPLER_LONGWORD_DECIMAL;
+ r5 = - 1;
+ } else {
+ result4 = r4;
+ r5 = 0;
+ }
+ if (r5 != 0) {
+ throw new RuntimeException("Unexpected underflow");
+ }
+
+ } else {
+
+ final long divideFactor = powerOfTenTable[THREE_X_LONGWORD_DECIMAL_DIGITS - diffScale];
+ final long multiplyFactor = powerOfTenTable[diffScale - TWO_X_LONGWORD_DECIMAL_DIGITS];
+
+ result0 = leftFast0;
+ result1 = leftFast1;
+ final long r2 =
+ leftFast2
+ - (rightFast0 % divideFactor) * multiplyFactor;
+ long r3;
+ if (r2 < 0) {
+ result2 = r2 + MULTIPLER_LONGWORD_DECIMAL;
+ r3 =
+ - (rightFast0 / divideFactor)
+ - (rightFast1 % divideFactor) * multiplyFactor
+ - 1;
+ } else {
+ result2 = r2;
+ r3 =
+ - (rightFast0 / divideFactor)
+ - (rightFast1 % divideFactor) * multiplyFactor;
+ }
+ long r4;
+ if (r3 < 0) {
+ result3 = r3 + MULTIPLER_LONGWORD_DECIMAL;
+ r4 =
+ - (rightFast1 / divideFactor)
+ - (rightFast2 % divideFactor) * multiplyFactor
+ - 1;
+ } else {
+ result3 = r3;
+ r4 =
+ - (rightFast1 / divideFactor)
+ - (rightFast2 % divideFactor) * multiplyFactor;
+ }
+ long r5;
+ if (r4 < 0) {
+ result4 = r4 + MULTIPLER_LONGWORD_DECIMAL;
+ r5 =
+ - (rightFast2 / divideFactor)
+ - 1;
+ } else {
+ result4 = r4;
+ r5 =
+ - (rightFast2 / divideFactor);
+ }
+ if (r5 != 0) {
+ throw new RuntimeException("Unexpected underflow");
+ }
+ }
+ } else {
+
+ // Since right has a longer digit tail and it doesn't move; we will shift the left digits
+ // as we do our addition into the result.
+
+ diffScale = rightScale - leftScale;
+ resultScale = rightScale;
+
+ if (diffScale < LONGWORD_DECIMAL_DIGITS) {
+
+ final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - diffScale];
+ final long multiplyFactor = powerOfTenTable[diffScale];
+
+ final long r0 =
+ (leftFast0 % divideFactor) * multiplyFactor
+ - rightFast0;
+ long r1;
+ if (r0 < 0) {
+ result0 = r0 + MULTIPLER_LONGWORD_DECIMAL;
+ r1 =
+ leftFast0 / divideFactor
+ + (leftFast1 % divideFactor) * multiplyFactor
+ - rightFast1
+ - 1;
+ } else {
+ result0 = r0;
+ r1 =
+ leftFast0 / divideFactor
+ + (leftFast1 % divideFactor) * multiplyFactor
+ - rightFast1;
+ }
+ long r2;
+ if (r1 < 0) {
+ result1 = r1 + MULTIPLER_LONGWORD_DECIMAL;
+ r2 =
+ leftFast1 / divideFactor
+ + (leftFast2 % divideFactor) * multiplyFactor
+ - rightFast2
+ - 1;
+ } else {
+ result1 = r1;
+ r2 =
+ leftFast1 / divideFactor
+ + (leftFast2 % divideFactor) * multiplyFactor
+ - rightFast2;
+ }
+ long r3;
+ if (r2 < 0) {
+ result2 = r2 + MULTIPLER_LONGWORD_DECIMAL;
+ r3 =
+ leftFast2 / divideFactor
+ - 1;
+ } else {
+ result2 = r2;
+ r3 =
+ leftFast2 / divideFactor;
+ }
+ long r4;
+ if (r3 < 0) {
+ result3 = r3 + MULTIPLER_LONGWORD_DECIMAL;
+ r4 = - 1;
+ } else {
+ result3 = r3;
+ r4 = 0;
+ }
+ if (r4 != 0) {
+ throw new RuntimeException("Unexpected underflow");
+ }
+
+ } else if (diffScale == LONGWORD_DECIMAL_DIGITS){
+
+ final long r0 =
+ - rightFast0;
+ long r1;
+ if (r0 < 0) {
+ result0 = r0 + MULTIPLER_LONGWORD_DECIMAL;
+ r1 =
+ leftFast0
+ - rightFast1
+ - 1;
+ } else {
+ result0 = r0;
+ r1 =
+ leftFast0
+ - rightFast1;
+ }
+ long r2;
+ if (r1 < 0) {
+ result1 = r1 + MULTIPLER_LONGWORD_DECIMAL;
+ r2 =
+ leftFast1
+ - rightFast2
+ - 1;
+ } else {
+ result1 = r1;
+ r2 =
+ leftFast1
+ - rightFast2;
+ }
+ long r3;
+ if (r2 < 0) {
+ result2 = r2 + MULTIPLER_LONGWORD_DECIMAL;
+ r3 =
+ leftFast2
+ - 1;
+ } else {
+ result2 = r2;
+ r3 =
+ leftFast2;
+ }
+ long r4;
+ if (r3 < 0) {
+ result3 = r3 + MULTIPLER_LONGWORD_DECIMAL;
+ r4 = - 1;
+ } else {
+ result3 = r3;
+ r4 = 0;
+ }
+ if (r4 != 0) {
+ throw new RuntimeException("Unexpected underflow");
+ }
+
+ } else if (diffScale < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ final long divideFactor = powerOfTenTable[TWO_X_LONGWORD_DECIMAL_DIGITS - diffScale];
+ final long multiplyFactor = powerOfTenTable[diffScale - LONGWORD_DECIMAL_DIGITS];
+
+ final long r0 =
+ - rightFast0;
+ long r1;
+ if (r0 < 0) {
+ result0 = r0 + MULTIPLER_LONGWORD_DECIMAL;
+ r1 =
+ (leftFast0 % divideFactor) * multiplyFactor
+ - rightFast1
+ - 1;
+ } else {
+ result0 = r0;
+ r1 =
+ (leftFast0 % divideFactor) * multiplyFactor
+ - rightFast1;
+ }
+ long r2;
+ if (r1 < 0) {
+ result1 = r1 + MULTIPLER_LONGWORD_DECIMAL;
+ r2 =
+ leftFast0 / divideFactor
+ + (leftFast1 % divideFactor) * multiplyFactor
+ - rightFast2
+ - 1;
+ } else {
+ result1 = r1;
+ r2 =
+ leftFast0 / divideFactor
+ + (leftFast1 % divideFactor) * multiplyFactor
+ - rightFast2;
+ }
+ long r3;
+ if (r2 < 0) {
+ result2 = r2 + MULTIPLER_LONGWORD_DECIMAL;
+ r3 =
+ leftFast1 / divideFactor
+ + (leftFast2 % divideFactor) * multiplyFactor
+ - 1;
+ } else {
+ result2 = r2;
+ r3 =
+ leftFast1 / divideFactor
+ + (leftFast2 % divideFactor) * multiplyFactor;
+ }
+ long r4;
+ if (r3 < 0) {
+ result3 = r3 + MULTIPLER_LONGWORD_DECIMAL;
+ r4 =
+ leftFast2 / divideFactor
+ - 1;
+ } else {
+ result3 = r3;
+ r4 =
+ leftFast2 / divideFactor;
+ }
+ if (r4 < 0) {
+ result4 = r4 + MULTIPLER_LONGWORD_DECIMAL;
+ } else {
+ result4 = r4;
+ }
+
+ } else if (diffScale == TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ final long r0 =
+ - rightFast0;
+ long r1;
+ if (r0 < 0) {
+ result0 = r0 + MULTIPLER_LONGWORD_DECIMAL;
+ r1 =
+ - rightFast1
+ - 1;
+ } else {
+ result0 = r0;
+ r1 =
+ - rightFast1;
+ }
+ long r2;
+ if (r1 < 0) {
+ result1 = r1 + MULTIPLER_LONGWORD_DECIMAL;
+ r2 =
+ leftFast0
+ - rightFast2
+ - 1;
+ } else {
+ result1 = r1;
+ r2 =
+ leftFast0
+ - rightFast2;
+ }
+ long r3;
+ if (r2 < 0) {
+ result2 = r2 + MULTIPLER_LONGWORD_DECIMAL;
+ r3 =
+ leftFast1
+ - 1;
+ } else {
+ result2 = r2;
+ r3 =
+ leftFast1;
+ }
+ long r4;
+ if (r3 < 0) {
+ result3 = r3 + MULTIPLER_LONGWORD_DECIMAL;
+ r4 =
+ leftFast2
+ - 1;
+ } else {
+ result3 = r3;
+ r4 =
+ leftFast2;
+ }
+ long r5;
+ if (r4 < 0) {
+ result4 = r4 + MULTIPLER_LONGWORD_DECIMAL;
+ r5 = - 1;
+ } else {
+ result4 = r4;
+ r5 = 0;
+ }
+ if (r5 != 0) {
+ throw new RuntimeException("Unexpected underflow");
+ }
+
+ } else {
+
+ final long divideFactor = powerOfTenTable[THREE_X_LONGWORD_DECIMAL_DIGITS - diffScale];
+ final long multiplyFactor = powerOfTenTable[diffScale - TWO_X_LONGWORD_DECIMAL_DIGITS];
+
+ final long r0 =
+ - rightFast0;
+ long r1;
+ if (r0 < 0) {
+ result0 = r0 + MULTIPLER_LONGWORD_DECIMAL;
+ r1 =
+ - rightFast1
+ - 1;
+ } else {
+ result0 = r0;
+ r1 =
+ - rightFast1;
+ }
+ long r2;
+ if (r1 < 0) {
+ result1 = r1 + MULTIPLER_LONGWORD_DECIMAL;
+ r2 =
+ (leftFast0 % divideFactor) * multiplyFactor
+ - rightFast2
+ - 1;
+ } else {
+ result1 = r1;
+ r2 =
+ (leftFast0 % divideFactor) * multiplyFactor
+ - rightFast2;
+ }
+ long r3;
+ if (r2 < 0) {
+ result2 = r2 + MULTIPLER_LONGWORD_DECIMAL;
+ r3 =
+ leftFast0 / divideFactor
+ + (leftFast1 % divideFactor) * multiplyFactor
+ - 1;
+ } else {
+ result2 = r2;
+ r3 =
+ leftFast0 / divideFactor
+ + (leftFast1 % divideFactor) * multiplyFactor;
+ }
+ long r4;
+ if (r3 < 0) {
+ result3 = r3 + MULTIPLER_LONGWORD_DECIMAL;
+ r4 =
+ leftFast1 / divideFactor
+ + (leftFast2 % divideFactor) * multiplyFactor
+ - 1;
+ } else {
+ result3 = r3;
+ r4 =
+ leftFast1 / divideFactor
+ + (leftFast2 % divideFactor) * multiplyFactor;
+ }
+ long r5;
+ if (r4 < 0) {
+ result4 = r4 + MULTIPLER_LONGWORD_DECIMAL;
+ r5 =
+ leftFast2 / divideFactor
+ - 1;
+ } else {
+ result4 = r4;
+ r5 =
+ leftFast2 / divideFactor;
+ }
+ if (r5 != 0) {
+ throw new RuntimeException("Unexpected underflow");
+ }
+ }
+ }
+
+ return
+ doFinishAddSubtractDifferentScale(
+ result0, result1, result2, result3, result4,
+ resultScale,
+ fastResult);
+ }
+
+ /**
+ * Handle decimal addition when the values have different scales.
+ */
+ private static boolean fastAddDifferentScale(
+ long leftFast0, long leftFast1, long leftFast2,
+ int leftIntegerDigitCount, int leftScale,
+ long rightFast0, long rightFast1, long rightFast2,
+ int rightIntegerDigitCount, int rightScale,
+ FastHiveDecimal fastResult) {
+
+ // Arrange so result* has a longer digit tail and it lines up; we will shift the shift* digits
+ // as we do our addition and them into the result.
+ long result0;
+ long result1;
+ long result2;
+
+ long shift0;
+ long shift1;
+ long shift2;
+
+ int diffScale;
+ int resultScale;
+
+ // Since addition is commutative, we can add in any order.
+ if (leftScale > rightScale) {
+
+ result0 = leftFast0;
+ result1 = leftFast1;
+ result2 = leftFast2;
+
+ shift0 = rightFast0;
+ shift1 = rightFast1;
+ shift2 = rightFast2;
+
+ diffScale = leftScale - rightScale;
+ resultScale = leftScale;
+ } else {
+
+ result0 = rightFast0;
+ result1 = rightFast1;
+ result2 = rightFast2;
+
+ shift0 = leftFast0;
+ shift1 = leftFast1;
+ shift2 = leftFast2;
+
+ diffScale = rightScale - leftScale;
+ resultScale = rightScale;
+ }
+
+ long result3 = 0;
+ long result4 = 0;
+
+ if (diffScale < LONGWORD_DECIMAL_DIGITS) {
+
+ final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - diffScale];
+ final long multiplyFactor = powerOfTenTable[diffScale];
+
+ final long r0 =
+ result0
+ + (shift0 % divideFactor) * multiplyFactor;
+ result0 =
+ r0 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r1 =
+ result1
+ + shift0 / divideFactor
+ + (shift1 % divideFactor) * multiplyFactor
+ + r0 / MULTIPLER_LONGWORD_DECIMAL;
+ result1 =
+ r1 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r2 =
+ result2
+ + shift1 / divideFactor
+ + (shift2 % divideFactor) * multiplyFactor
+ + r1 / MULTIPLER_LONGWORD_DECIMAL;
+ result2 =
+ r2 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r3 =
+ shift2 / divideFactor
+ + r2 / MULTIPLER_LONGWORD_DECIMAL;
+ result3 =
+ r3 % MULTIPLER_LONGWORD_DECIMAL;
+
+ } else if (diffScale == LONGWORD_DECIMAL_DIGITS){
+
+ final long r1 =
+ result1
+ + shift0;
+ result1 =
+ r1 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r2 =
+ result2
+ + shift1
+ + r1 / MULTIPLER_LONGWORD_DECIMAL;
+ result2 =
+ r2 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r3 =
+ shift2
+ + r2 / MULTIPLER_LONGWORD_DECIMAL;
+ result3 =
+ r3 % MULTIPLER_LONGWORD_DECIMAL;
+ result4 = r3 / MULTIPLER_LONGWORD_DECIMAL;
+
+ } else if (diffScale < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ final long divideFactor = powerOfTenTable[TWO_X_LONGWORD_DECIMAL_DIGITS - diffScale];
+ final long multiplyFactor = powerOfTenTable[diffScale - LONGWORD_DECIMAL_DIGITS];
+
+ final long r1 =
+ result1
+ + (shift0 % divideFactor) * multiplyFactor;
+ result1 =
+ r1 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r2 =
+ result2
+ + shift0 / divideFactor
+ + (shift1 % divideFactor) * multiplyFactor
+ + r1 / MULTIPLER_LONGWORD_DECIMAL;
+ result2 =
+ r2 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r3 =
+ shift1 / divideFactor
+ + (shift2 % divideFactor) * multiplyFactor
+ + r2 / MULTIPLER_LONGWORD_DECIMAL;
+ result3 =
+ r3 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r4 =
+ shift2 / divideFactor
+ + r3 / MULTIPLER_LONGWORD_DECIMAL;
+ result4 =
+ r4 % MULTIPLER_LONGWORD_DECIMAL;
+
+ } else if (diffScale == TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ final long r2 =
+ result2
+ + shift0;
+ result2 =
+ r2 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r3 =
+ shift1
+ + r2 / MULTIPLER_LONGWORD_DECIMAL;
+ result3 =
+ r3 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r4 =
+ shift2
+ + r3 / MULTIPLER_LONGWORD_DECIMAL;
+ result4 =
+ r4 % MULTIPLER_LONGWORD_DECIMAL;
+
+ } else {
+
+ final long divideFactor = powerOfTenTable[THREE_X_LONGWORD_DECIMAL_DIGITS - diffScale];
+ final long multiplyFactor = powerOfTenTable[diffScale - TWO_X_LONGWORD_DECIMAL_DIGITS];
+
+ final long r2 =
+ result2
+ + (shift0 % divideFactor) * multiplyFactor;
+ result2 =
+ r2 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r3 =
+ shift0 / divideFactor
+ + (shift1 % divideFactor) * multiplyFactor
+ + r2 / MULTIPLER_LONGWORD_DECIMAL;
+ result3 =
+ r3 % MULTIPLER_LONGWORD_DECIMAL;
+ final long r4 =
+ shift1 / divideFactor
+ + (shift2 % divideFactor) * multiplyFactor
+ + r3 / MULTIPLER_LONGWORD_DECIMAL;
+ result4 =
+ r4 % MULTIPLER_LONGWORD_DECIMAL;
+ if (shift2 / divideFactor != 0) {
+ throw new RuntimeException("Unexpected overflow");
+ }
+
+ }
+
+ return
+ doFinishAddSubtractDifferentScale(
+ result0, result1, result2, result3, result4,
+ resultScale,
+ fastResult);
+ }
+
+ private static boolean doAddDifferentScale(
+ int leftSignum, long leftFast0, long leftFast1, long leftFast2,
+ int leftIntegerDigitCount, int leftScale,
+ int rightSignum, long rightFast0, long rightFast1, long rightFast2,
+ int rightIntegerDigitCount, int rightScale,
+ FastHiveDecimal fastResult) {
+
+ if (leftSignum == rightSignum) {
+ if (!fastAddDifferentScale(
+ leftFast0, leftFast1, leftFast2,
+ leftIntegerDigitCount, leftScale,
+ rightFast0, rightFast1, rightFast2,
+ rightIntegerDigitCount, rightScale,
+ fastResult)) {
+ return false;
+ }
+ // Sign stays the same.
+ fastResult.fastSignum = leftSignum;
+ } else {
+
+ // Just compare the magnitudes (i.e. signums set to 1).
+ int compareTo =
+ fastCompareTo(
+ 1,
+ leftFast0, leftFast1, leftFast2, leftScale,
+ 1,
+ rightFast0, rightFast1, rightFast2, rightScale);
+ if (compareTo == 0) {
+ // They cancel each other.
+ fastResult.fastSignum = 0;
+ fastResult.fast0 = 0;
+ fastResult.fast1 = 0;
+ fastResult.fast2 = 0;
+ fastResult.fastScale = 0;
+ return true;
+ }
+
+ if (compareTo == 1) {
+ if (!fastSubtractDifferentScale(
+ leftFast0, leftFast1, leftFast2,
+ leftIntegerDigitCount, leftScale,
+ rightFast0, rightFast1, rightFast2,
+ rightIntegerDigitCount, rightScale,
+ fastResult)) {
+ throw new RuntimeException("Unexpected overflow");
+ }
+ fastResult.fastSignum = leftSignum;
+ } else {
+ if (!fastSubtractDifferentScale(
+ rightFast0, rightFast1, rightFast2,
+ rightIntegerDigitCount, rightScale,
+ leftFast0, leftFast1, leftFast2,
+ leftIntegerDigitCount, leftScale,
+ fastResult)) {
+ throw new RuntimeException("Unexpected overflow");
+ }
+ fastResult.fastSignum = rightSignum;
+ }
+ }
+
+ final int resultTrailingZeroCount =
+ fastTrailingDecimalZeroCount(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ fastResult.fastIntegerDigitCount, fastResult.fastScale);
+ if (resultTrailingZeroCount > 0) {
+ doFastScaleDown(
+ fastResult,
+ resultTrailingZeroCount,
+ fastResult);
+ if (fastResult.fastSignum == 0) {
+ fastResult.fastScale = 0;
+ } else {
+ fastResult.fastScale -= resultTrailingZeroCount;
+ }
+ }
+
+ return true;
+ }
+
+ public static boolean fastAdd(
+ FastHiveDecimal fastLeft,
+ FastHiveDecimal fastRight,
+ FastHiveDecimal fastResult) {
+ return fastAdd(
+ fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2,
+ fastLeft.fastIntegerDigitCount, fastLeft.fastScale,
+ fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2,
+ fastRight.fastIntegerDigitCount, fastRight.fastScale,
+ fastResult);
+ }
+
+ /**
+ * Add the two decimals.
+ *
+ * NOTE: Scale Determination for Addition/Subtraction
+ *
+ * One could take the Math.min of the scales and adjust the operand with the lower scale have a
+ * scale = higher scale.
+ *
+ * But this does not seem to work with decimals with widely varying scales as these:
+ *
+ * 598575157855521918987423259.94094 dec1 (int digits 27,scale 5)
+ * + 0.0000000000006711991169422033 dec2 (int digits 0, scale 28)
+ *
+ * Trying to make dec1 to have a scale of 28 (i.e. by adding trailing zeroes) would exceed
+ * MAX_PRECISION (int digits 27 + 28 > 38).
+ *
+ * In this example we need to make sure we have enough integer digit room in the result to
+ * handle dec1's digits. In order to maintain that, we will need to get rid of lower
+ * fractional digits of dec2. But when do we do that?
+ *
+ * OldHiveDecimal.add does the full arithmetic add with all the digits using BigDecimal and
+ * then adjusts the result to fit in MAX_PRECISION, etc.
+ *
+ * If we try to do pre-rounding dec2 it is problematic. We'd need to know if there is a carry in
+ * the arithmetic in order to know at which scale to do the rounding. This gets complicated.
+ *
+ * So, the simplest thing is to emulate what OldHiveDecimal does and do the full digit addition
+ * and then fit the result afterwards.
+ *
+ * @param leftSignum The left sign (-1, 0, or +1)
+ * @param leftFast0 The left word 0 of reprentation
+ * @param leftFast1 word 1
+ * @param leftFast2 word 2
+ * @param leftIntegerDigitCount The left number of integer digits
+ * @param leftScale the left scale
+ * @param rightSignum The right sign (-1, 0, or +1)
+ * @param rightFast0 The right word 0 of reprentation
+ * @param rightFast1 word 1
+ * @param rightFast2 word 2
+ * @param rightIntegerDigitCount The right number of integer digits
+ * @param rightScale the right scale
+ * @param fastResult an object to reuse
+ * @return True if the addition was successful; Otherwise, false is returned on overflow.
+ */
+ public static boolean fastAdd(
+ int leftSignum, long leftFast0, long leftFast1, long leftFast2,
+ int leftIntegerDigitCount, int leftScale,
+ int rightSignum, long rightFast0, long rightFast1, long rightFast2,
+ int rightIntegerDigitCount, int rightScale,
+ FastHiveDecimal fastResult) {
+
+ if (rightSignum == 0) {
+ fastResult.fastSet(leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale);
+ return true;
+ }
+ if (leftSignum == 0) {
+ fastResult.fastSet(rightSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale);
+ return true;
+ }
+
+ if (leftScale == rightScale) {
+ return doAddSameScale(
+ leftSignum, leftFast0, leftFast1, leftFast2,
+ rightSignum, rightFast0, rightFast1, rightFast2,
+ leftScale,
+ fastResult);
+ } else {
+ return doAddDifferentScale(
+ leftSignum, leftFast0, leftFast1, leftFast2,
+ leftIntegerDigitCount, leftScale,
+ rightSignum, rightFast0, rightFast1, rightFast2,
+ rightIntegerDigitCount, rightScale,
+ fastResult);
+ }
+ }
+
+ public static boolean fastSubtract(
+ FastHiveDecimal fastLeft,
+ FastHiveDecimal fastRight,
+ FastHiveDecimal fastResult) {
+ return fastSubtract(
+ fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2,
+ fastLeft.fastIntegerDigitCount, fastLeft.fastScale,
+ fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2,
+ fastRight.fastIntegerDigitCount, fastRight.fastScale,
+ fastResult);
+ }
+
+ public static boolean fastSubtract(
+ int leftSignum, long leftFast0, long leftFast1, long leftFast2,
+ int leftIntegerDigitCount, int leftScale,
+ int rightSignum, long rightFast0, long rightFast1, long rightFast2,
+ int rightIntegerDigitCount, int rightScale,
+ FastHiveDecimal fastResult) {
+
+ if (rightSignum == 0) {
+ fastResult.fastSet(leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale);
+ return true;
+ }
+ final int flippedDecSignum = (rightSignum == 1 ? -1 : 1);
+ if (leftSignum == 0) {
+ fastResult.fastSet(flippedDecSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale);
+ return true;
+ }
+
+ if (leftScale == rightScale) {
+ return doAddSameScale(
+ leftSignum, leftFast0, leftFast1, leftFast2,
+ flippedDecSignum, rightFast0, rightFast1, rightFast2,
+ leftScale,
+ fastResult);
+ } else {
+ return doAddDifferentScale(
+ leftSignum, leftFast0, leftFast1, leftFast2,
+ leftIntegerDigitCount, leftScale,
+ flippedDecSignum, rightFast0, rightFast1, rightFast2,
+ rightIntegerDigitCount, rightScale,
+ fastResult);
+ }
+ }
+
+ //************************************************************************************************
+ // Decimal Multiply.
+
+ private static boolean doMultiply(
+ int leftSignum, long leftFast0, long leftFast1, long leftFast2,
+ int leftIntegerDigitCount, int leftScale,
+ int rightSignum, long rightFast0, long rightFast1, long rightFast2,
+ int rightIntegerDigitCount, int rightScale,
+ FastHiveDecimal fastResult) {
+
+ // Set signum before; if result is zero, fastMultiply will set signum to 0.
+ fastResult.fastSignum = (leftSignum == rightSignum ? 1 : -1);
+ int resultScale = leftScale + rightScale;
+
+ /*
+ * For multiplicands with scale 0, trim trailing zeroes.
+ */
+ if (leftScale == 0) {
+
+ // Pretend like it has fractional digits so we can get the trailing zero count.
+ final int leftTrailingZeroCount =
+ fastTrailingDecimalZeroCount(
+ leftFast0, leftFast1, leftFast2,
+ 0, leftIntegerDigitCount);
+ if (leftTrailingZeroCount > 0) {
+ doFastScaleDown(
+ leftFast0, leftFast1, leftFast2, leftTrailingZeroCount, fastResult);
+ resultScale -= leftTrailingZeroCount;
+ leftFast0 = fastResult.fast0;
+ leftFast1 = fastResult.fast1;
+ leftFast2 = fastResult.fast2;
+ }
+ }
+ if (rightScale == 0) {
+
+ // Pretend like it has fractional digits so we can get the trailing zero count.
+ final int rightTrailingZeroCount =
+ fastTrailingDecimalZeroCount(
+ rightFast0, rightFast1, rightFast2,
+ 0, rightIntegerDigitCount);
+ if (rightTrailingZeroCount > 0) {
+ doFastScaleDown(
+ rightFast0, rightFast1, rightFast2, rightTrailingZeroCount, fastResult);
+ resultScale -= rightTrailingZeroCount;
+ rightFast0 = fastResult.fast0;
+ rightFast1 = fastResult.fast1;
+ rightFast2 = fastResult.fast2;
+ }
+ }
+
+ boolean largeOverflow =
+ !fastMultiply5x5HalfWords(
+ leftFast0, leftFast1, leftFast2,
+ rightFast0, rightFast1, rightFast2,
+ fastResult);
+ if (largeOverflow) {
+ return false;
+ }
+
+ if (fastResult.fastSignum == 0) {
+ fastResult.fastScale = 0;
+ return true;
+ }
+
+ if (resultScale < 0) {
+ if (-resultScale >= HiveDecimal.MAX_SCALE) {
+ return false;
+ }
+ if (!fastScaleUp(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2, -resultScale,
+ fastResult)) {
+ return false;
+ }
+ resultScale = 0;
+ }
+
+ int precision;
+ if (fastResult.fast2 != 0) {
+ precision = TWO_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast2);
+ } else if (fastResult.fast1 != 0) {
+ precision = LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast1);
+ } else {
+ precision = fastLongWordPrecision(fastResult.fast0);
+ }
+
+ int integerDigitCount = Math.max(0, precision - resultScale);
+ if (integerDigitCount > HiveDecimal.MAX_PRECISION) {
+ // Integer is too large -- cannot recover by trimming fractional digits.
+ return false;
+ }
+
+ if (precision > HiveDecimal.MAX_PRECISION || resultScale > HiveDecimal.MAX_SCALE) {
+
+ // Trim off lower fractional digits but with NO ROUNDING.
+
+ final int maxScale = HiveDecimal.MAX_SCALE - integerDigitCount;
+ final int scaleDown = resultScale - maxScale;
+ if (!fastScaleDownNoRound(
+ fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ scaleDown,
+ fastResult)) {
+ // Round fractional must be 0. Not allowed to throw away digits.
+ return false;
+ }
+ resultScale -= scaleDown;
+ }
+ fastResult.fastScale = resultScale;
+
+ // This assume no round up...
+ fastResult.fastIntegerDigitCount = integerDigitCount;
+
+ if (fastResult.fastScale > HiveDecimal.MAX_SCALE) {
+ // We are not allowed to lose digits in multiply to be compatible with OldHiveDecimal
+ // behavior, so overflow.
+ // CONSIDER: Does it make sense to be so restrictive. If we just did repeated addition,
+ // it would succeed...
+ return false;
+ }
+ final int resultTrailingZeroCount =
+ fastTrailingDecimalZeroCount(
+ fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ fastResult.fastIntegerDigitCount, fastResult.fastScale);
+ if (resultTrailingZeroCount > 0) {
+ doFastScaleDown(
+ fastResult,
+ resultTrailingZeroCount,
+ fastResult);
+ if (fastResult.fastSignum == 0) {
+ fastResult.fastScale = 0;
+ } else {
+ fastResult.fastScale -= resultTrailingZeroCount;
+ }
+ }
+
+ return true;
+ }
+
+ public static boolean fastMultiply5x5HalfWords(
+ FastHiveDecimal fastLeft,
+ FastHiveDecimal fastRight,
+ FastHiveDecimal fastResult) {
+ return
+ fastMultiply5x5HalfWords(
+ fastLeft.fast0, fastLeft.fast1, fastLeft.fast2,
+ fastRight.fast0, fastRight.fast1, fastRight.fast2,
+ fastResult);
+ }
+
+ /**
+ * Fast decimal multiplication on two decimals that have been already scaled and whose results
+ * will fit in 38 digits.
+ *
+ * The caller is responsible checking for overflow within the highword and determining
+ * if scale down appropriate.
+ *
+ * @return Returns false if the multiplication resulted in large overflow. Values in result are
+ * undefined in that case.
+ */
+ public static boolean fastMultiply5x5HalfWords(
+ long left0, long left1, long left2,
+ long right0, long right1, long right2,
+ FastHiveDecimal fastResult) {
+
+ long product;
+
+ final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL;
+
+ final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL;
+
+ // v[0]
+ product =
+ halfRight0 * halfLeft0;
+ final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0].
+ product =
+ halfRight0
+ * halfLeft1
+ + halfRight1
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[2]
+ product =
+ halfRight0
+ * halfLeft2
+ + halfRight1
+ * halfLeft1
+ + halfRight2
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[3]
+ product =
+ halfRight0
+ * halfLeft3
+ + halfRight1
+ * halfLeft2
+ + halfRight2
+ * halfLeft1
+ + halfRight3
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[4]
+ product =
+ halfRight0
+ * halfLeft4
+ + halfRight1
+ * halfLeft3
+ + halfRight2
+ * halfLeft2
+ + halfRight3
+ * halfLeft1
+ + halfRight4
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+
+ // v[5] is not calculated since high integer is always 0 for our decimals.
+
+ // These remaining combinations below definitely result in overflow.
+ if ((halfRight4 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0 || halfLeft1 != 0))
+ || (halfRight3 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0))
+ || (halfRight2 != 0 && (halfLeft4 != 0 || halfLeft3 != 0))
+ || (halfRight1 != 0 && halfLeft4 != 0)) {
+ return false;
+ }
+
+
+ final long result0 = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0;
+ final long result1 = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2;
+ final long result2 = product;
+
+ if (result0 == 0 && result1 == 0 && result2 == 0) {
+ fastResult.fastSignum = 0;
+ }
+ fastResult.fast0 = result0;
+ fastResult.fast1 = result1;
+ fastResult.fast2 = result2;
+
+ return true;
+ }
+
+ public static boolean fastMultiplyFullInternal(
+ FastHiveDecimal fastLeft,
+ FastHiveDecimal fastRight,
+ long[] result) {
+ return
+ fastMultiplyFullInternal(
+ fastLeft.fast0, fastLeft.fast1, fastLeft.fast2,
+ fastRight.fast0, fastRight.fast1, fastRight.fast2,
+ result);
+ }
+
+ /**
+ * Fast decimal multiplication on two decimals that have been already scaled and whose results
+ * will fit in 38 digits.
+ *
+ * The caller is responsible checking for overflow within the highword and determining
+ * if scale down appropriate.
+ *
+ * @return Returns false if the multiplication resulted in large overflow. Values in result are
+ * undefined in that case.
+ */
+ public static boolean fastMultiply5x5HalfWords(
+ long left0, long left1, long left2,
+ long right0, long right1, long right2,
+ long[] result) {
+
+ long product;
+
+ final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL;
+
+ final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL;
+
+ // v[0]
+ product =
+ halfRight0 * halfLeft0;
+ final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0].
+ product =
+ halfRight0
+ * halfLeft1
+ + halfRight1
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[2]
+ product =
+ halfRight0
+ * halfLeft2
+ + halfRight1
+ * halfLeft1
+ + halfRight2
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[3]
+ product =
+ halfRight0
+ * halfLeft3
+ + halfRight1
+ * halfLeft2
+ + halfRight2
+ * halfLeft1
+ + halfRight3
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[4]
+ product =
+ halfRight0
+ * halfLeft4
+ + halfRight1
+ * halfLeft3
+ + halfRight2
+ * halfLeft2
+ + halfRight3
+ * halfLeft1
+ + halfRight4
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+
+ // v[5] is not calculated since high integer is always 0 for our decimals.
+
+ // These remaining combinations below definitely result in overflow.
+ if ((halfRight4 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0 || halfLeft1 != 0))
+ || (halfRight3 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0))
+ || (halfRight2 != 0 && (halfLeft4 != 0 || halfLeft3 != 0))
+ || (halfRight1 != 0 && halfLeft4 != 0)) {
+ return false;
+ }
+
+ result[0] = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0;
+ result[1] = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2;
+ result[2] = product;
+
+ return true;
+ }
+
+ /**
+ * Fast decimal multiplication on two decimals whose results are permitted to go beyond
+ * 38 digits to the maximum possible 76 digits. The caller is responsible for scaling and
+ * rounding the results back to 38 or fewer digits.
+ *
+ * The caller is responsible for determining the signum.
+ *
+ * @param left0
+ * @param left1
+ * @param left2
+ * @param right0
+ * @param right1
+ * @param right2
+ * @param result This full result has 5 longs.
+ * @return Returns false if the multiplication resulted in an overflow. Values in result are
+ * undefined in that case.
+ */
+ public static boolean fastMultiplyFullInternal(
+ long left0, long left1, long left2,
+ long right0, long right1, long right2,
+ long[] result) {
+ assert (result.length == 5);
+ if (result.length != 5) {
+ throw new IllegalArgumentException("Expecting result array length = 5");
+ }
+
+ long product;
+
+ final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL;
+
+ final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL;
+
+ // v[0]
+ product =
+ halfRight0 * halfLeft0;
+ final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0].
+ product =
+ halfRight0
+ * halfLeft1
+ + halfRight1
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[2]
+ product =
+ halfRight0
+ * halfLeft2
+ + halfRight1
+ * halfLeft1
+ + halfRight2
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[3]
+ product =
+ halfRight0
+ * halfLeft3
+ + halfRight1
+ * halfLeft2
+ + halfRight2
+ * halfLeft1
+ + halfRight3
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[4]
+ product =
+ halfRight0
+ * halfLeft4
+ + halfRight1
+ * halfLeft3
+ + halfRight2
+ * halfLeft2
+ + halfRight3
+ * halfLeft1
+ + halfRight4
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z4 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[5] -- since integer #5 is always 0, some products here are not included.
+ product =
+ halfRight1
+ * halfLeft4
+ + halfRight2
+ * halfLeft3
+ + halfRight3
+ * halfLeft2
+ + halfRight4
+ * halfLeft1
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z5 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[6] -- since integer #5 is always 0, some products here are not included.
+ product =
+ halfRight2
+ * halfLeft4
+ + halfRight3
+ * halfLeft3
+ + halfRight4
+ * halfLeft2
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z6 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[7] -- since integer #5 is always 0, some products here are not included.
+ product =
+ halfRight3
+ * halfLeft4
+ + halfRight4
+ * halfLeft3
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z7 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[8] -- since integer #5 is always 0, some products here are not included.
+ product =
+ halfRight4
+ * halfLeft4
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z8 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[9] -- since integer #5 is always 0, some products here are not included.
+ product =
+ (product / MULTIPLER_INTWORD_DECIMAL);
+ if (product > FULL_MAX_HIGHWORD_DECIMAL) {
+ return false;
+ }
+
+ result[0] = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0;
+ result[1] = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2;
+ result[2] = (long) z5 * MULTIPLER_INTWORD_DECIMAL + (long) z4;
+ result[3] = (long) z7 * MULTIPLER_INTWORD_DECIMAL + (long) z6;
+ result[4] = product * MULTIPLER_INTWORD_DECIMAL + (long) z8;
+
+ return true;
+ }
+
+ /**
+ * Fast decimal multiplication on two decimals whose results are permitted to go beyond
+ * 38 digits to the maximum possible 76 digits. The caller is responsible for scaling and
+ * rounding the results back to 38 or fewer digits.
+ *
+ * The caller is responsible for determining the signum.
+ *
+ * @param result This full result has 5 longs.
+ * @return Returns false if the multiplication resulted in an overflow. Values in result are
+ * undefined in that case.
+ */
+ public static boolean fastMultiply5x6HalfWords(
+ long left0, long left1, long left2,
+ long right0, long right1, long right2,
+ long[] result) {
+
+ if (result.length != 6) {
+ throw new RuntimeException("Expecting result array length = 6");
+ }
+
+ long product;
+
+ final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfRight5 = right2 / MULTIPLER_INTWORD_DECIMAL;
+
+ final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL;
+ final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL;
+
+ // v[0]
+ product =
+ halfRight0 * halfLeft0;
+ final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0].
+ product =
+ halfRight0
+ * halfLeft1
+ + halfRight1
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[2]
+ product =
+ halfRight0
+ * halfLeft2
+ + halfRight1
+ * halfLeft1
+ + halfRight2
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[3]
+ product =
+ halfRight0
+ * halfLeft3
+ + halfRight1
+ * halfLeft2
+ + halfRight2
+ * halfLeft1
+ + halfRight3
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[4]
+ product =
+ halfRight0
+ * halfLeft4
+ + halfRight1
+ * halfLeft3
+ + halfRight2
+ * halfLeft2
+ + halfRight3
+ * halfLeft1
+ + halfRight4
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z4 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[5] -- since left integer #5 is always 0, some products here are not included.
+ product =
+ halfRight1
+ * halfLeft4
+ + halfRight2
+ * halfLeft3
+ + halfRight3
+ * halfLeft2
+ + halfRight4
+ * halfLeft1
+ + halfRight5
+ * halfLeft0
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z5 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[6] -- since left integer #5 is always 0, some products here are not included.
+ product =
+ halfRight2
+ * halfLeft4
+ + halfRight3
+ * halfLeft3
+ + halfRight4
+ * halfLeft2
+ + halfRight5
+ * halfLeft1
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z6 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[7] -- since left integer #5 is always 0, some products here are not included.
+ product =
+ halfRight3
+ * halfLeft4
+ + halfRight4
+ * halfLeft3
+ + halfRight5
+ * halfLeft2
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z7 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[8] -- since left integer #5 is always 0, some products here are not included.
+ product =
+ halfRight4
+ * halfLeft4
+ + halfRight5
+ * halfLeft3
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z8 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[9] -- since left integer #5 is always 0, some products here are not included.
+ product =
+ halfRight5
+ * halfLeft4
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ final int z9 = (int) (product % MULTIPLER_INTWORD_DECIMAL);
+
+ // v[10] -- since left integer #5 is always 0, some products here are not included.
+ product =
+ + (product / MULTIPLER_INTWORD_DECIMAL);
+ if (product > MULTIPLER_INTWORD_DECIMAL) {
+ return false;
+ }
+
+ result[0] = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0;
+ result[1] = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2;
+ result[2] = (long) z5 * MULTIPLER_INTWORD_DECIMAL + (long) z4;
+ result[3] = (long) z7 * MULTIPLER_INTWORD_DECIMAL + (long) z6;
+ result[4] = (long) z9 * MULTIPLER_INTWORD_DECIMAL + (long) z8;
+ result[5] = product;
+
+ return true;
+ }
+
+ public static boolean fastMultiply(
+ FastHiveDecimal fastLeft,
+ FastHiveDecimal fastRight,
+ FastHiveDecimal fastResult) {
+ return fastMultiply(
+ fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2,
+ fastLeft.fastIntegerDigitCount, fastLeft.fastScale,
+ fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2,
+ fastRight.fastIntegerDigitCount, fastRight.fastScale,
+ fastResult);
+ }
+
+ public static boolean fastMultiply(
+ int leftSignum, long leftFast0, long leftFast1, long leftFast2,
+ int leftIntegerDigitCount, int leftScale,
+ int rightSignum, long rightFast0, long rightFast1, long rightFast2,
+ int rightIntegerDigitCount, int rightScale,
+ FastHiveDecimal fastResult) {
+
+ if (leftSignum == 0 || rightSignum == 0) {
+ fastResult.fastReset();
+ return true;
+ }
+
+ return doMultiply(
+ leftSignum, leftFast0, leftFast1, leftFast2,
+ leftIntegerDigitCount, leftScale,
+ rightSignum, rightFast0, rightFast1, rightFast2,
+ rightIntegerDigitCount, rightScale,
+ fastResult);
+ }
+
+ //************************************************************************************************
+ // Decimal Division / Remainder.
+
+ /**
+ * EXPERMIMENTAL: Division when divisor fits in a single decimal longword.
+ *
+ * @return remainderSubexpr2
+ */
+ private static long doSingleWordQuotient(
+ long leftFast0, long leftFast1, long leftFast2,
+ long rightFast0,
+ FastHiveDecimal fastResult) {
+
+ long quotient2;
+ long quotient1;
+ long quotient0;
+
+ long remainderSubexpr2;
+
+ if (leftFast2 == 0 && leftFast1 == 0) {
+ quotient2 = 0;
+ quotient1 = 0;
+ quotient0 =
+ leftFast0 / rightFast0;
+ final long k0 =
+ leftFast0 - quotient0 * rightFast0;
+ remainderSubexpr2 =
+ k0 * MULTIPLER_LONGWORD_DECIMAL;
+ } else if (leftFast2 == 0) {
+ // leftFast1 != 0.
+ quotient2 = 0;
+ quotient1 =
+ leftFast1 / rightFast0;
+ final long k1 =
+ leftFast1 - quotient1 * rightFast0;
+ final long quotientSubexpr0 =
+ k1 * MULTIPLER_LONGWORD_DECIMAL
+ + leftFast0;
+ quotient0 =
+ quotientSubexpr0 / rightFast0;
+ final long k0 =
+ quotientSubexpr0 - quotient0 * rightFast0;
+ remainderSubexpr2 =
+ k0 * MULTIPLER_LONGWORD_DECIMAL;
+ } else if (leftFast1 == 0){
+ // leftFast2 != 0 && leftFast1 == 0.
+ quotient2 =
+ leftFast2 / rightFast0;
+ quotient1 = 0;
+ quotient0 =
+ leftFast0 / rightFast0;
+ final long k0 =
+ leftFast0 - quotient0 * rightFast0;
+ remainderSubexpr2 =
+ k0 * MULTIPLER_LONGWORD_DECIMAL;
+ } else {
+ quotient2 =
+ leftFast2 / rightFast0;
+ final long k2 =
+ leftFast2 - quotient2 * rightFast0;
+ final long quotientSubexpr1 =
+ k2 * MULTIPLER_LONGWORD_DECIMAL
+ + leftFast1;
+ quotient1 =
+ quotientSubexpr1 / rightFast0;
+ final long k1 =
+ quotientSubexpr1 - quotient1 * rightFast0;
+ final long quotientSubexpr0 =
+ k1 * MULTIPLER_LONGWORD_DECIMAL;
+ quotient0 =
+ quotientSubexpr0 / rightFast0;
+ final long k0 =
+ quotientSubexpr0 - quotient0 * rightFast0;
+ remainderSubexpr2 =
+ k0 * MULTIPLER_LONGWORD_DECIMAL;
+ }
+
+ fastResult.fast0 = quotient0;
+ fastResult.fast1 = quotient1;
+ fastResult.fast2 = quotient2;
+
+ return remainderSubexpr2;
+ }
+
+ private static int doSingleWordRemainder(
+ long leftFast0, long leftFast1, long leftFast2,
+ long rightFast0,
+ long remainderSubexpr2,
+ FastHiveDecimal fastResult) {
+
+ int remainderDigitCount;
+
+ long remainder2;
+ long remainder1;
+ long remainder0;
+
+ if (remainderSubexpr2 == 0) {
+ remainder2 = 0;
+ remainder1 = 0;
+ remainder0 = 0;
+ remainderDigitCount = 0;
+ } else {
+ remainder2 =
+ remainderSubexpr2 / rightFast0;
+ final long k2 =
+ remainderSubexpr2 - remainder2 * rightFast0;
+ if (k2 == 0) {
+ remainder1 = 0;
+ remainder0 = 0;
+ remainderDigitCount =
+ LONGWORD_DECIMAL_DIGITS - fastLongWordTrailingZeroCount(remainder2);
+ } else {
+ final long remainderSubexpr1 =
+ k2 * MULTIPLER_LONGWORD_DECIMAL;
+ long remainderSubexpr0;
+ remainder1 =
+ remainderSubexpr1 / rightFast0;
+ final long k1 =
+ remainderSubexpr1 - remainder1 * rightFast0;
+ if (k1 == 0) {
+ remainder0 = 0;
+ remainderDigitCount =
+ LONGWORD_DECIMAL_DIGITS
+ + LONGWORD_DECIMAL_DIGITS - fastLongWordTrailingZeroCount(remainder1);
+ } else {
+ remainderSubexpr0 =
+ k2 * MULTIPLER_LONGWORD_DECIMAL;
+
+ remainder0 =
+ remainderSubexpr0 / rightFast0;
+ remainderDigitCount =
+ TWO_X_LONGWORD_DECIMAL_DIGITS
+ + LONGWORD_DECIMAL_DIGITS - fastLongWordTrailingZeroCount(remainder0);
+ }
+ }
+ }
+
+ fastResult.fast0 = remainder0;
+ fastResult.fast1 = remainder1;
+ fastResult.fast2 = remainder2;
+
+ return remainderDigitCount;
+ }
+
+ // EXPERIMENT
+ private static boolean fastSingleWordDivision(
+ int leftSignum, long leftFast0, long leftFast1, long leftFast2, int leftScale,
+ int rightSignum, long rightFast0, int rightScale,
+ FastHiveDecimal fastResult) {
+
+ long remainderSubexpr2 =
+ doSingleWordQuotient(
+ leftFast0, leftFast1, leftFast2,
+ rightFast0,
+ fastResult);
+
+ long quotient0 = fastResult.fast0;
+ long quotient1 = fastResult.fast1;
+ long quotient2 = fastResult.fast2;
+
+ int quotientDigitCount;
+ if (quotient2 != 0) {
+ quotientDigitCount = fastLongWordPrecision(quotient2);
+ } else if (quotient1 != 0) {
+ quotientDigitCount = fastLongWordPrecision(quotient1);
+ } else {
+ quotientDigitCount = fastLongWordPrecision(quotient0);
+ }
+
+ int remainderDigitCount =
+ doSingleWordRemainder(
+ leftFast0, leftFast1, leftFast2,
+ rightFast0,
+ remainderSubexpr2,
+ fastResult);
+
+ long remainder0 = fastResult.fast0;
+ long remainder1 = fastResult.fast1;
+ long remainder2 = fastResult.fast2;
+
+ fastResult.fast0 = quotient0;
+ fastResult.fast1 = quotient1;
+ fastResult.fast2 = quotient2;
+
+ final int quotientScale = leftScale + rightScale;
+
+ if (remainderDigitCount == 0) {
+ fastResult.fastScale = quotientScale;
+ } else {
+ int resultScale = quotientScale + remainderDigitCount;
+
+ int adjustedQuotientDigitCount;
+ if (quotientScale > 0) {
+ adjustedQuotientDigitCount = Math.max(0, quotientDigitCount - quotientScale);
+ } else {
+ adjustedQuotientDigitCount = quotientDigitCount;
+ }
+ final int maxScale = HiveDecimal.MAX_SCALE - adjustedQuotientDigitCount;
+
+ int scale = Math.min(resultScale, maxScale);
+
+ int remainderScale;
+ remainderScale = Math.min(remainderDigitCount, maxScale - quotientScale);
+ if (remainderScale > 0) {
+ if (quotientDigitCount > 0) {
+ // Make room for remainder.
+ fastScaleUp(
+ fastResult,
+ remainderScale,
+ fastResult);
+ }
+ // Copy in remainder digits... which start at the top of remainder2.
+ if (remainderScale < LONGWORD_DECIMAL_DIGITS) {
+ final long remainderDivisor2 = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - remainderScale];
+ fastResult.fast0 += (remainder2 / remainderDivisor2);
+ } else if (remainderScale == LONGWORD_DECIMAL_DIGITS) {
+ fastResult.fast0 = remainder2;
+ } else if (remainderScale < TWO_X_LONGWORD_DECIMAL_DIGITS) {
+ final long remainderDivisor2 = powerOfTenTable[remainderScale - LONGWORD_DECIMAL_DIGITS];
+ fastResult.fast1 += (remainder2 / remainderDivisor2);
+ fastResult.fast0 = remainder1;
+ } else if (remainderScale == TWO_X_LONGWORD_DECIMAL_DIGITS) {
+ fastResult.fast1 = remainder2;
+ fastResult.fast0 = remainder1;
+ }
+ }
+
+ // UNDONE: Method is still under development.
+ fastResult.fastScale = scale;
+
+ // UNDONE: Trim trailing zeroes...
+ }
+
+ return true;
+ }
+
+ public static boolean fastDivide(
+ FastHiveDecimal fastLeft,
+ FastHiveDecimal fastRight,
+ FastHiveDecimal fastResult) {
+ return fastDivide(
+ fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2,
+ fastLeft.fastIntegerDigitCount, fastLeft.fastScale,
+ fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2,
+ fastRight.fastIntegerDigitCount, fastRight.fastScale,
+ fastResult);
+ }
+
+ public static boolean fastDivide(
+ int leftSignum, long leftFast0, long leftFast1, long leftFast2,
+ int leftIntegerDigitCount, int leftScale,
+ int rightSignum, long rightFast0, long rightFast1, long rightFast2,
+ int rightIntegerDigitCount, int rightScale,
+ FastHiveDecimal fastResult) {
+
+ // Arithmetic operations reset the results.
+ fastResult.fastReset();
+
+ if (rightSignum == 0) {
+ // Division by 0.
+ return false;
+ }
+ if (leftSignum == 0) {
+ // Zero result.
+ return true;
+ }
+
+ /*
+ if (rightFast1 == 0 && rightFast2 == 0) {
+ return fastSingleWordDivision(
+ leftSignum, leftFast0, leftFast1, leftFast2, leftScale,
+ rightSignum, rightFast0, rightScale,
+ fastResult);
+ }
+ */
+
+ BigDecimal denominator =
+ fastBigDecimalValue(
+ leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale);
+ BigDecimal divisor =
+ fastBigDecimalValue(
+ rightSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale);
+ BigDecimal quotient =
+ denominator.divide(divisor, HiveDecimal.MAX_SCALE, BigDecimal.ROUND_HALF_UP);
+
+ if (!fastSetFromBigDecimal(
+ quotient,
+ true,
+ fastResult)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ public static boolean fastRemainder(
+ int leftSignum, long leftFast0, long leftFast1, long leftFast2,
+ int leftIntegerDigitCount, int leftScale,
+ int rightSignum, long rightFast0, long rightFast1, long rightFast2,
+ int rightIntegerDigitCount, int rightScale,
+ FastHiveDecimal fastResult) {
+
+ // Arithmetic operations reset the results.
+ fastResult.fastReset();
+
+ if (rightSignum == 0) {
+ // Division by 0.
+ return false;
+ }
+ if (leftSignum == 0) {
+ // Zero result.
+ return true;
+ }
+
+ BigDecimal denominator =
+ fastBigDecimalValue(
+ leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale);
+ BigDecimal divisor =
+ fastBigDecimalValue(
+ rightSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale);
+ BigDecimal remainder =
+ denominator.remainder(divisor);
+ fastResult.fastReset();
+ if (!fastSetFromBigDecimal(
+ remainder,
+ true,
+ fastResult)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ public static boolean fastPow(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int exponent,
+ FastHiveDecimal fastResult) {
+
+ // Arithmetic operations (re)set the results.
+ fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+
+ if (exponent < 0) {
+ // UNDONE: Currently, negative exponent is not supported.
+ return false;
+ }
+
+ for (int e = 1; e < exponent; e++) {
+ if (!doMultiply(
+ fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ fastResult.fastIntegerDigitCount, fastResult.fastScale,
+ fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2,
+ fastResult.fastIntegerDigitCount, fastResult.fastScale,
+ fastResult)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ //************************************************************************************************
+ // Decimal String Formatting.
+
+ public static String fastToFormatString(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int formatScale) {
+ byte[] scratchBuffer = new byte[FAST_SCRATCH_BUFFER_LEN_TO_BYTES];
+ final int index =
+ doFastToFormatBytes(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ formatScale,
+ scratchBuffer);
+ return
+ new String(scratchBuffer, index, FAST_SCRATCH_BUFFER_LEN_TO_BYTES - index);
+ }
+
+ //************************************************************************************************
+ // Decimal String Formatting.
+
+ public static String fastToFormatString(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int formatScale,
+ byte[] scratchBuffer) {
+ final int index =
+ doFastToBytes(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale, formatScale,
+ scratchBuffer);
+ return new String(scratchBuffer, index, scratchBuffer.length - index);
+ }
+
+ public static int fastToFormatBytes(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int formatScale,
+ byte[] scratchBuffer) {
+ return
+ doFastToFormatBytes(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ formatScale,
+ scratchBuffer);
+ }
+
+ public static int doFastToFormatBytes(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale,
+ int formatScale,
+ byte[] scratchBuffer) {
+
+ // NOTE: OldHiveDecimal.toFormatString returns decimal strings with more than > 38 digits!
+
+ if (formatScale >= fastScale) {
+ return
+ doFastToBytes(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale, formatScale,
+ scratchBuffer);
+ } else {
+ FastHiveDecimal fastTemp = new FastHiveDecimal();
+ if (!fastRound(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale,
+ formatScale, BigDecimal.ROUND_HALF_UP,
+ fastTemp)) {
+ return 0;
+ }
+ return
+ doFastToBytes(
+ fastTemp.fastSignum, fastTemp.fast0, fastTemp.fast1, fastTemp.fast2,
+ fastTemp.fastIntegerDigitCount, fastTemp.fastScale, formatScale,
+ scratchBuffer);
+ }
+ }
+
+ public static String fastToString(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale) {
+ return doFastToString(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale, fastTrailingZeroesScale);
+ }
+
+ public static String fastToString(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale,
+ byte[] scratchBuffer) {
+ return doFastToString(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale, fastTrailingZeroesScale,
+ scratchBuffer);
+ }
+
+ public static String fastToDigitsOnlyString(
+ long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount) {
+ byte[] scratchBuffer = new byte[FAST_SCRATCH_BUFFER_LEN_TO_BYTES];
+ final int index =
+ doFastToDigitsOnlyBytes(
+ fast0, fast1, fast2,
+ fastIntegerDigitCount,
+ scratchBuffer);
+ return
+ new String(scratchBuffer, index, FAST_SCRATCH_BUFFER_LEN_TO_BYTES - index);
+ }
+
+ public static int fastToBytes(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale,
+ byte[] scratchBuffer) {
+ return doFastToBytes(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale, fastTrailingZeroesScale,
+ scratchBuffer);
+ }
+
+ private static String doFastToString(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale) {
+ byte[] scratchBuffer = new byte[FAST_SCRATCH_BUFFER_LEN_TO_BYTES];
+ final int index =
+ doFastToBytes(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale, fastTrailingZeroesScale,
+ scratchBuffer);
+ return
+ new String(
+ scratchBuffer, index, FAST_SCRATCH_BUFFER_LEN_TO_BYTES - index);
+ }
+
+ private static String doFastToString(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale,
+ byte[] scratchBuffer) {
+ final int index =
+ doFastToBytes(
+ fastSignum, fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale, fastTrailingZeroesScale,
+ scratchBuffer);
+ return new String(scratchBuffer, index, scratchBuffer.length - index);
+ }
+
+ private static int doFastToBytes(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale,
+ byte[] scratchBuffer) {
+
+ int index = scratchBuffer.length - 1;
+
+ int trailingZeroCount =
+ (fastTrailingZeroesScale != -1 ? fastTrailingZeroesScale - fastScale : 0);
+ // Virtual trailing zeroes.
+ if (trailingZeroCount > 0) {
+ for (int i = 0; i < trailingZeroCount; i++) {
+ scratchBuffer[index--] = BYTE_DIGIT_ZERO;
+ }
+ }
+
+ // Scale fractional digits, dot, integer digits.
+
+ final int scale = fastScale;
+
+ final boolean isZeroFast1AndFast2 = (fast1 == 0 && fast2 == 0);
+ final boolean isZeroFast2 = (fast2 == 0);
+
+ int lowerLongwordScale = 0;
+ int middleLongwordScale = 0;
+ int highLongwordScale = 0;
+ long longWord = fast0;
+ if (scale > 0) {
+
+ // Fraction digits from lower longword.
+
+ lowerLongwordScale = Math.min(scale, LONGWORD_DECIMAL_DIGITS);
+
+ for (int i = 0; i < lowerLongwordScale; i++) {
+ scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10);
+ longWord /= 10;
+ }
+ if (lowerLongwordScale == LONGWORD_DECIMAL_DIGITS) {
+ longWord = fast1;
+ }
+
+ if (scale > LONGWORD_DECIMAL_DIGITS) {
+
+ // Fraction digits continue into middle longword.
+
+ middleLongwordScale = Math.min(scale - LONGWORD_DECIMAL_DIGITS, LONGWORD_DECIMAL_DIGITS);
+ for (int i = 0; i < middleLongwordScale; i++) {
+ scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10);
+ longWord /= 10;
+ }
+ if (middleLongwordScale == LONGWORD_DECIMAL_DIGITS) {
+ longWord = fast2;
+ }
+
+ if (scale > TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ // Fraction digit continue into highest longword.
+
+ highLongwordScale = scale - TWO_X_LONGWORD_DECIMAL_DIGITS;
+ for (int i = 0; i < highLongwordScale; i++) {
+ scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10);
+ longWord /= 10;
+ }
+ }
+ }
+ scratchBuffer[index--] = BYTE_DOT;
+ } else if (trailingZeroCount > 0) {
+ scratchBuffer[index--] = BYTE_DOT;
+ }
+
+ // Integer digits; stop on zeroes above.
+
+ boolean atLeastOneIntegerDigit = false;
+ if (scale <= LONGWORD_DECIMAL_DIGITS) {
+
+ // Handle remaining lower long word digits as integer digits.
+
+ final int remainingLowerLongwordDigits = LONGWORD_DECIMAL_DIGITS - lowerLongwordScale;
+ for (int i = 0; i < remainingLowerLongwordDigits; i++) {
+ scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10);
+ atLeastOneIntegerDigit = true;
+ longWord /= 10;
+ if (longWord == 0 && isZeroFast1AndFast2) {
+ // Suppress leading zeroes.
+ break;
+ }
+ }
+ if (isZeroFast1AndFast2) {
+ if (!atLeastOneIntegerDigit) {
+ scratchBuffer[index--] = BYTE_DIGIT_ZERO;
+ }
+ if (fastSignum == -1) {
+ scratchBuffer[index--] = BYTE_MINUS;
+ }
+ return index + 1;
+ }
+ longWord = fast1;
+ }
+
+ if (scale <= TWO_X_LONGWORD_DECIMAL_DIGITS) {
+
+ // Handle remaining middle long word digits.
+
+ final int remainingMiddleLongwordDigits = LONGWORD_DECIMAL_DIGITS - middleLongwordScale;
+
+ for (int i = 0; i < remainingMiddleLongwordDigits; i++) {
+ scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10);
+ atLeastOneIntegerDigit = true;
+ longWord /= 10;
+ if (longWord == 0 && isZeroFast2) {
+ // Suppress leading zeroes.
+ break;
+ }
+ }
+ if (isZeroFast2) {
+ if (!atLeastOneIntegerDigit) {
+ scratchBuffer[index--] = BYTE_DIGIT_ZERO;
+ }
+ if (fastSignum == -1) {
+ scratchBuffer[index--] = BYTE_MINUS;
+ }
+ return index + 1;
+ }
+ longWord = fast2;
+ }
+
+ final int remainingHighwordDigits = HIGHWORD_DECIMAL_DIGITS - highLongwordScale;
+
+ for (int i = 0; i < remainingHighwordDigits; i++) {
+ scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10);
+ atLeastOneIntegerDigit = true;
+ longWord /= 10;
+ if (longWord == 0) {
+ // Suppress leading zeroes.
+ break;
+ }
+ }
+ if (!atLeastOneIntegerDigit) {
+ scratchBuffer[index--] = BYTE_DIGIT_ZERO;
+ }
+ if (fastSignum == -1) {
+ scratchBuffer[index--] = BYTE_MINUS;
+ }
+ return index + 1;
+ }
+
+ public static int fastToDigitsOnlyBytes(
+ long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount,
+ byte[] scratchBuffer) {
+ return doFastToDigitsOnlyBytes(
+ fast0, fast1, fast2,
+ fastIntegerDigitCount,
+ scratchBuffer);
+ }
+
+ private static int doFastToDigitsOnlyBytes(
+ long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount,
+ byte[] scratchBuffer) {
+
+ int index = scratchBuffer.length - 1;
+
+ // Just digits.
+
+ final boolean isZeroFast1AndFast2 = (fast1 == 0 && fast2 == 0);
+ final boolean isZeroFast2 = (fast2 == 0);
+
+ boolean atLeastOneIntegerDigit = false;
+ long longWord = fast0;
+ for (int i = 0; i < LONGWORD_DECIMAL_DIGITS; i++) {
+ scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10);
+ atLeastOneIntegerDigit = true;
+ longWord /= 10;
+ if (longWord == 0 && isZeroFast1AndFast2) {
+ // Suppress leading zeroes.
+ break;
+ }
+ }
+ if (isZeroFast1AndFast2) {
+ if (!atLeastOneIntegerDigit) {
+ scratchBuffer[index--] = BYTE_DIGIT_ZERO;
+ }
+ return index + 1;
+ }
+
+ longWord = fast1;
+
+ for (int i = 0; i < LONGWORD_DECIMAL_DIGITS; i++) {
+ scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10);
+ atLeastOneIntegerDigit = true;
+ longWord /= 10;
+ if (longWord == 0 && isZeroFast2) {
+ // Suppress leading zeroes.
+ break;
+ }
+ }
+ if (isZeroFast2) {
+ if (!atLeastOneIntegerDigit) {
+ scratchBuffer[index--] = BYTE_DIGIT_ZERO;
+ }
+ return index + 1;
+ }
+
+ longWord = fast2;
+
+ for (int i = 0; i < HIGHWORD_DECIMAL_DIGITS; i++) {
+ scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10);
+ atLeastOneIntegerDigit = true;
+ longWord /= 10;
+ if (longWord == 0) {
+ // Suppress leading zeroes.
+ break;
+ }
+ }
+ if (!atLeastOneIntegerDigit) {
+ scratchBuffer[index--] = BYTE_DIGIT_ZERO;
+ }
+ return index + 1;
+ }
+
+ //************************************************************************************************
+ // Decimal Validation.
+
+ public static boolean fastIsValid(FastHiveDecimal fastDec) {
+ return fastIsValid(
+ fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2,
+ fastDec.fastIntegerDigitCount, fastDec.fastScale);
+ }
+
+ public static boolean fastIsValid(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+ boolean isValid;
+ if (fastSignum == 0) {
+ isValid = (fast0 == 0 && fast1 == 0 && fast2 == 0 && fastIntegerDigitCount == 0 && fastScale == 0);
+ if (!isValid) {
+ System.out.println("FAST_IS_VALID signum 0 but other fields not");
+ }
+ } else {
+ isValid = (
+ (fast0 >= 0 && fast0 <= MAX_LONGWORD_DECIMAL) &&
+ (fast1 >= 0 && fast1 <= MAX_LONGWORD_DECIMAL) &&
+ (fast2 >= 0 && fast2 <= MAX_HIGHWORD_DECIMAL));
+ if (!isValid) {
+ System.out.println("FAST_IS_VALID fast0 .. fast2 out of range");
+ } else {
+ if (fastScale < 0 || fastScale > HiveDecimal.MAX_SCALE) {
+ System.out.println("FAST_IS_VALID fastScale " + fastScale + " out of range");
+ isValid = false;
+ } else if (fastIntegerDigitCount < 0 || fastIntegerDigitCount > HiveDecimal.MAX_PRECISION) {
+ System.out.println("FAST_IS_VALID fastIntegerDigitCount " + fastIntegerDigitCount + " out of range");
+ isValid = false;
+ } else if (fastIntegerDigitCount + fastScale > HiveDecimal.MAX_PRECISION) {
+ System.out.println("FAST_IS_VALID exceeds max precision: fastIntegerDigitCount " + fastIntegerDigitCount + " and fastScale " + fastScale);
+ isValid = false;
+ } else {
+ // Verify integerDigitCount given fastScale.
+ final int rawPrecision = fastRawPrecision(fastSignum, fast0, fast1, fast2);
+ if (fastIntegerDigitCount > 0) {
+ if (rawPrecision != fastIntegerDigitCount + fastScale) {
+ System.out.println("FAST_IS_VALID integer case: rawPrecision " + rawPrecision +
+ " fastIntegerDigitCount " + fastIntegerDigitCount +
+ " fastScale " + fastScale);
+ isValid = false;
+ }
+ } else {
+ if (rawPrecision > fastScale) {
+ System.out.println("FAST_IS_VALID fraction only case: rawPrecision " + rawPrecision +
+ " fastIntegerDigitCount " + fastIntegerDigitCount +
+ " fastScale " + fastScale);
+ isValid = false;
+ }
+ }
+ if (isValid) {
+ final int trailingZeroCount =
+ fastTrailingDecimalZeroCount(
+ fast0, fast1, fast2,
+ fastIntegerDigitCount, fastScale);
+ if (trailingZeroCount != 0) {
+ System.out.println("FAST_IS_VALID exceeds max precision: trailingZeroCount != 0");
+ isValid = false;
+ }
+ }
+ }
+ }
+ }
+
+ if (!isValid) {
+ System.out.println("FAST_IS_VALID fast0 " + fast0);
+ System.out.println("FAST_IS_VALID fast1 " + fast1);
+ System.out.println("FAST_IS_VALID fast2 " + fast2);
+ System.out.println("FAST_IS_VALID fastIntegerDigitCount " + fastIntegerDigitCount);
+ System.out.println("FAST_IS_VALID fastScale " + fastScale);
+ }
+ return isValid;
+ }
+
+ public static void fastRaiseInvalidException(
+ FastHiveDecimal fastResult) {
+ throw new RuntimeException(
+ "Invalid fast decimal " +
+ " fastSignum " + fastResult.fastSignum + " fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2 +
+ " fastIntegerDigitCount " + fastResult.fastIntegerDigitCount + " fastScale " + fastResult.fastScale +
+ " stack trace: " + getStackTraceAsSingleLine(Thread.currentThread().getStackTrace()));
+ }
+
+ public static void fastRaiseInvalidException(
+ FastHiveDecimal fastResult,
+ String parameters) {
+ throw new RuntimeException(
+ "Parameters: " + parameters + " --> " +
+ "Invalid fast decimal " +
+ " fastSignum " + fastResult.fastSignum + " fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2 +
+ " fastIntegerDigitCount " + fastResult.fastIntegerDigitCount + " fastScale " + fastResult.fastScale +
+ " stack trace: " + getStackTraceAsSingleLine(Thread.currentThread().getStackTrace()));
+ }
+
+ //************************************************************************************************
+ // Decimal Debugging.
+
+ static final int STACK_LENGTH_LIMIT = 20;
+ public static String getStackTraceAsSingleLine(StackTraceElement[] stackTrace) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Stack trace: ");
+ int length = stackTrace.length;
+ boolean isTruncated = false;
+ if (length > STACK_LENGTH_LIMIT) {
+ length = STACK_LENGTH_LIMIT;
+ isTruncated = true;
+ }
+ for (int i = 0; i < length; i++) {
+ if (i > 0) {
+ sb.append(", ");
+ }
+ sb.append(stackTrace[i]);
+ }
+ if (isTruncated) {
+ sb.append(", ...");
+ }
+
+ return sb.toString();
+ }
+
+ public static String displayBytes(byte[] bytes, int start, int length) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = start; i < start + length; i++) {
+ sb.append(String.format("\\%03d", (int) (bytes[i] & 0xff)));
+ }
+ return sb.toString();
+ }
+}
\ No newline at end of file
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java
new file mode 100644
index 0000000000000000000000000000000000000000..53684e7ab1b8d473ac7618ab2c830c226175a18c
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.type;
+
+import org.apache.commons.lang.StringUtils;
+
+public abstract class HiveBaseChar {
+ protected String value;
+
+ protected HiveBaseChar() {
+ }
+
+ /**
+ * Sets the string value to a new value, obeying the max length defined for this object.
+ * @param val new value
+ */
+ public void setValue(String val, int maxLength) {
+ value = HiveBaseChar.enforceMaxLength(val, maxLength);
+ }
+
+ public void setValue(HiveBaseChar val, int maxLength) {
+ setValue(val.value, maxLength);
+ }
+
+ public static String enforceMaxLength(String val, int maxLength) {
+ if (val == null) {
+ return null;
+ }
+ String value = val;
+
+ if (maxLength > 0) {
+ int valLength = val.codePointCount(0, val.length());
+ if (valLength > maxLength) {
+ // Truncate the excess chars to fit the character length.
+ // Also make sure we take supplementary chars into account.
+ value = val.substring(0, val.offsetByCodePoints(0, maxLength));
+ }
+ }
+ return value;
+ }
+
+ public static String getPaddedValue(String val, int maxLength) {
+ if (val == null) {
+ return null;
+ }
+ if (maxLength < 0) {
+ return val;
+ }
+
+ int valLength = val.codePointCount(0, val.length());
+ if (valLength > maxLength) {
+ return enforceMaxLength(val, maxLength);
+ }
+
+ if (maxLength > valLength) {
+ // Make sure we pad the right amount of spaces; valLength is in terms of code points,
+ // while StringUtils.rpad() is based on the number of java chars.
+ int padLength = val.length() + (maxLength - valLength);
+ val = StringUtils.rightPad(val, padLength);
+ }
+ return val;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public int getCharacterLength() {
+ return value.codePointCount(0, value.length());
+ }
+
+ @Override
+ public int hashCode() {
+ return getValue().hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return getValue();
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveChar.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveChar.java
new file mode 100644
index 0000000000000000000000000000000000000000..66aa524bb2c3b0835cdb8af3001ec6d96b5ae67c
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveChar.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.type;
+
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * HiveChar.
+ * String values will be padded to full char length.
+ * Character legnth, comparison, hashCode should ignore trailing spaces.
+ */
+public class HiveChar extends HiveBaseChar
+ implements Comparable {
+
+ public static final int MAX_CHAR_LENGTH = 255;
+
+ public HiveChar() {
+ }
+
+ public HiveChar(String val, int len) {
+ setValue(val, len);
+ }
+
+ public HiveChar(HiveChar hc, int len) {
+ setValue(hc.value, len);
+ }
+
+ /**
+ * Set char value, padding or truncating the value to the size of len parameter.
+ */
+ public void setValue(String val, int len) {
+ super.setValue(HiveBaseChar.getPaddedValue(val, len), -1);
+ }
+
+ public void setValue(String val) {
+ setValue(val, -1);
+ }
+
+ public String getStrippedValue() {
+ return StringUtils.stripEnd(value, " ");
+ }
+
+ public String getPaddedValue() {
+ return value;
+ }
+
+ public int getCharacterLength() {
+ String strippedValue = getStrippedValue();
+ return strippedValue.codePointCount(0, strippedValue.length());
+ }
+
+ public String toString() {
+ return getPaddedValue();
+ }
+
+ public int compareTo(HiveChar rhs) {
+ if (rhs == this) {
+ return 0;
+ }
+ return this.getStrippedValue().compareTo(rhs.getStrippedValue());
+ }
+
+ public boolean equals(Object rhs) {
+ if (rhs == this) {
+ return true;
+ }
+ if (rhs == null || rhs.getClass() != getClass()) {
+ return false;
+ }
+ return this.getStrippedValue().equals(((HiveChar) rhs).getStrippedValue());
+ }
+
+ public int hashCode() {
+ return getStrippedValue().hashCode();
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
new file mode 100644
index 0000000000000000000000000000000000000000..690e95762444a17fa601e03f68302fe8903c62a3
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
@@ -0,0 +1,1506 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.type;
+
+import java.util.Arrays;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+/**
+ * HiveDecimal is a decimal data type with a maximum precision and scale.
+ *
+ * It is the Hive DECIMAL data type.
+ *
+ * The scale is the number of fractional decimal digits. The digits after the dot. It is limited
+ * to 38 (MAX_SCALE).
+ *
+ * The precision is the integer (or whole-number) decimal digits plus fractional decimal digits.
+ * It is limited to a total of 38 digits (MAX_PRECISION).
+ *
+ * Hive syntax for declaring DECIMAL has 3 forms:
+ *
+ * {@code
+ * DECIMAL // Use the default precision/scale.}
+ *
+ * {@code
+ * DECIMAL(precision) // Use the default scale.}
+ *
+ * Use DECIMAL instead of DOUBLE when exact numeric accuracy is required. Not all decimal numbers
+ * (radix 10) are exactly representable in the binary (radix 2 based) floating point type DOUBLE and
+ * cause accuracy anomalies (i.e. wrong results). See the Internet for more details.
+ *
+ * HiveDecimal is implemented as a classic Java immutable object. All operations on HiveDecimal
+ * that produce a different value will create a new HiveDecimal object.
+ *
+ * Decimals are physically stored without any extra leading or trailing zeroes. The scale of
+ * a decimal is the number of non-trailing zero fractional digits.
+ *
+ * Math operations on decimals typically cause the scale to change as a result of the math and
+ * from trailing fractional digit elimination.
+ *
+ * Typically, Hive, when it wants to make sure a result decimal fits in the column decimal's
+ * precision/scale it calls enforcePrecisionScale. That method will scale down or trim off
+ * result fractional digits if necessary with rounding when the column has a smaller scale.
+ * And, it will also indicate overflow when the decimal has exceeded the column's maximum precision.
+ *
+ * NOTE: When Hive gets ready to serialize a decimal into text or binary, it usually sometimes
+ * wants trailing fractional zeroes. See the special notes for toFormatString and
+ * bigIntegerBytesScaled for details.
+ *
+ * ------------------------------------- Version 2 ------------------------------------------------
+ *
+ * This is the 2nd major version of HiveDecimal called V2. The previous version has been
+ * renamed to HiveDecimalV1 and is kept as a test and behavior reference.
+ *
+ * For good performance we do not represent the decimal using a BigDecimal object like the previous
+ * version V1 did. Using Java objects to represent our decimal incurs too high a penalty
+ * for memory allocations and general logic.
+ *
+ * The original V1 public methods and fields are annotated with @HiveDecimalVersionV1; new public
+ * methods and fields are annotated with @HiveDecimalVersionV2.
+ *
+ */
+public final class HiveDecimal extends FastHiveDecimal implements Comparable {
+
+ /*
+ * IMPLEMENTATION NOTE:
+ * We implement HiveDecimal with the mutable FastHiveDecimal class. That class uses
+ * protected on all its methods so they will not be visible in the HiveDecimal class.
+ *
+ * So even if one casts to FastHiveDecimal, you shouldn't be able to violate the immutability
+ * of a HiveDecimal class.
+ */
+
+ //TODO : We should ideally get the values of these constants from serdeConstants for consistency
+ @HiveDecimalVersionV1
+ public static final int MAX_PRECISION = 38;
+ @HiveDecimalVersionV1
+ public static final int MAX_SCALE = 38;
+
+ /**
+ * Default precision/scale when user doesn't specify in the column metadata, such as
+ * decimal and decimal(8).
+ */
+ @HiveDecimalVersionV1
+ public static final int USER_DEFAULT_PRECISION = 10;
+ @HiveDecimalVersionV1
+ public static final int USER_DEFAULT_SCALE = 0;
+
+ /**
+ * Default precision/scale when system is not able to determine them, such as in case
+ * of a non-generic udf.
+ */
+ @HiveDecimalVersionV1
+ public static final int SYSTEM_DEFAULT_PRECISION = 38;
+ @HiveDecimalVersionV1
+ public static final int SYSTEM_DEFAULT_SCALE = 18;
+
+ /**
+ * Common values.
+ */
+ @HiveDecimalVersionV1
+ public static final HiveDecimal ZERO = HiveDecimal.create(0);
+ @HiveDecimalVersionV1
+ public static final HiveDecimal ONE = HiveDecimal.create(1);
+
+ /**
+ * ROUND_FLOOR:
+ *
+ * Round towards negative infinity.
+ *
+ * The Hive function is FLOOR.
+ *
+ * Positive numbers: The round fraction is thrown away.
+ *
+ * (Example here rounds at scale 0)
+ * Value FLOOR
+ * 0.3 0
+ * 2 2
+ * 2.1 2
+ *
+ * Negative numbers: If there is a round fraction, throw it away and subtract 1.
+ *
+ * (Example here rounds at scale 0)
+ * Value FLOOR
+ * -0.3 -1
+ * -2 -2
+ * -2.1 -3
+ */
+ @HiveDecimalVersionV1
+ public static final int ROUND_FLOOR = BigDecimal.ROUND_FLOOR;
+
+ /**
+ * ROUND_CEILING:
+ *
+ * Round towards positive infinity.
+ *
+ * The Hive function is CEILING.
+ *
+ * Positive numbers: If there is a round fraction, throw it away and add 1
+ *
+ * (Example here rounds at scale 0)
+ * Value CEILING
+ * 0.3 1
+ * 2 2
+ * 2.1 3
+ *
+ * Negative numbers: The round fraction is thrown away.
+ *
+ * (Example here rounds at scale 0)
+ * Value CEILING
+ * -0.3 0
+ * -2 -2
+ * -2.1 -2
+ */
+ @HiveDecimalVersionV1
+ public static final int ROUND_CEILING = BigDecimal.ROUND_CEILING;
+
+ /**
+ * ROUND_HALF_UP:
+ *
+ * Round towards "nearest neighbor" unless both neighbors are equidistant then round up.
+ *
+ * The Hive function is ROUND.
+ *
+ * For result, throw away round fraction. If the round fraction is >= 0.5, then add 1 when
+ * positive and subtract 1 when negative. So, the sign is irrelevant.
+ *
+ * (Example here rounds at scale 0)
+ * Value ROUND Value ROUND
+ * 0.3 0 -0.3 0
+ * 2 2 -2 -2
+ * 2.1 2 -2.1 -2
+ * 2.49 2 -2.49 -2
+ * 2.5 3 -2.5 -3
+ *
+ */
+ @HiveDecimalVersionV1
+ public static final int ROUND_HALF_UP = BigDecimal.ROUND_HALF_UP;
+
+ /**
+ * ROUND_HALF_EVEN:
+ * Round towards the "nearest neighbor" unless both neighbors are equidistant, then round
+ * towards the even neighbor.
+ *
+ * The Hive function is BROUND.
+ *
+ * Known as Banker’s Rounding.
+ *
+ * When you add values rounded with ROUND_HALF_UP you have a bias that grows as you add more
+ * numbers. Banker's Rounding is a way to minimize that bias. It rounds toward the nearest
+ * even number when the fraction is 0.5 exactly. In table below, notice that 2.5 goes DOWN to
+ * 2 (even) but 3.5 goes UP to 4 (even), etc.
+ *
+ * So, the sign is irrelevant.
+ *
+ * (Example here rounds at scale 0)
+ * Value BROUND Value BROUND
+ * 0.49 0 -0.49 0
+ * 0.5 0 -0.5 0
+ * 0.51 1 -0.51 -1
+ * 1.5 2 -1.5 -2
+ * 2.5 2 -2.5 -2
+ * 2.51 3 -2.51 -3
+ * 3.5 4 -3.5 -4
+ * 4.5 4 -4.5 -4
+ * 4.51 5 -4.51 -5
+ *
+ */
+ @HiveDecimalVersionV1
+ public static final int ROUND_HALF_EVEN = BigDecimal.ROUND_HALF_EVEN;
+
+ //-----------------------------------------------------------------------------------------------
+ // Constructors are marked private; use create methods.
+ //-----------------------------------------------------------------------------------------------
+
+ private HiveDecimal() {
+ super();
+ }
+
+ private HiveDecimal(HiveDecimal dec) {
+ super(dec);
+ }
+
+ private HiveDecimal(FastHiveDecimal fastDec) {
+ super(fastDec);
+ }
+
+ private HiveDecimal(int fastSignum, FastHiveDecimal fastDec) {
+ super(fastSignum, fastDec);
+ }
+
+ private HiveDecimal(
+ int fastSignum, long fast0, long fast1, long fast2,
+ int fastIntegerDigitCount, int fastScale) {
+ super(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale);
+ }
+
+ //-----------------------------------------------------------------------------------------------
+ // Create methods.
+ //-----------------------------------------------------------------------------------------------
+
+ /**
+ * Create a HiveDecimal from a FastHiveDecimal object. Used by HiveDecimalWritable.
+ * @param fastDec the value to set
+ * @return new hive decimal
+ */
+ @HiveDecimalVersionV2
+ public static HiveDecimal createFromFast(FastHiveDecimal fastDec) {
+ return new HiveDecimal(fastDec);
+ }
+
+ /**
+ * Create a HiveDecimal from BigDecimal object.
+ *
+ * A BigDecimal object has a decimal scale.
+ *
+ * We will have overflow if BigDecimal's integer part exceed MAX_PRECISION digits or
+ * 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1.
+ *
+ * When the BigDecimal value's precision exceeds MAX_PRECISION and there are fractional digits
+ * because of scale > 0, then lower digits are trimmed off with rounding to meet the
+ * MAX_PRECISION requirement.
+ *
+ * Also, BigDecimal supports negative scale -- which means multiplying the value by 10^abs(scale).
+ * And, BigDecimal allows for a non-zero scale for zero. We normalize that so zero always has
+ * scale 0.
+ *
+ * @param bigDecimal the value to set
+ * @return The HiveDecimal with the BigDecimal's value adjusted down to a maximum precision.
+ * Otherwise, null is returned for overflow.
+ */
+ @HiveDecimalVersionV1
+ public static HiveDecimal create(BigDecimal bigDecimal) {
+ return create(bigDecimal, true);
+ }
+
+ /**
+ * Same as the above create method, except fractional digit rounding can be turned off.
+ * @param bigDecimal the value to set
+ * @param allowRounding True requires all of the bigDecimal value be converted to the decimal
+ * without loss of precision.
+ * @return
+ */
+ @HiveDecimalVersionV1
+ public static HiveDecimal create(BigDecimal bigDecimal, boolean allowRounding) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromBigDecimal(
+ bigDecimal, allowRounding)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Creates a HiveDecimal from a BigInteger's value with a scale of 0.
+ *
+ * We will have overflow if BigInteger exceed MAX_PRECISION digits or
+ * 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1.
+ *
+ * @param bigInteger the value to set
+ * @return A HiveDecimal object with the exact BigInteger's value.
+ * Otherwise, null is returned on overflow.
+ */
+ @HiveDecimalVersionV1
+ public static HiveDecimal create(BigInteger bigInteger) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromBigInteger(
+ bigInteger)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Creates a HiveDecimal from a BigInteger's value with a specified scale.
+ *
+ * We will have overflow if BigInteger exceed MAX_PRECISION digits or
+ * 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1.
+ *
+ * The resulting decimal will have fractional digits when the specified scale is greater than 0.
+ *
+ * When the BigInteger's value's precision exceeds MAX_PRECISION and there are fractional digits
+ * because of scale > 0, then lower digits are trimmed off with rounding to meet the
+ * MAX_PRECISION requirement.
+ *
+ * @param bigInteger the value to set
+ * @param scale the scale to set
+ * @return A HiveDecimal object with the BigInteger's value adjusted for scale.
+ * Otherwise, null is returned on overflow.
+ */
+ @HiveDecimalVersionV1
+ public static HiveDecimal create(BigInteger bigInteger, int scale) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromBigIntegerAndScale(
+ bigInteger, scale)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Create a HiveDecimal by parsing a whole string.
+ *
+ * We support parsing a decimal with an exponent because the previous version
+ * (i.e. OldHiveDecimal) uses the BigDecimal parser and was able to.
+ *
+ * @param string the string to parse
+ * @return a new hive decimal
+ */
+ @HiveDecimalVersionV1
+ public static HiveDecimal create(String string) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromString(
+ string, true)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Same as the method above, except blanks before and after are tolerated.
+ * @param string the string to parse
+ * @param trimBlanks True specifies leading and trailing blanks are to be ignored.
+ * @return a new hive decimal
+ */
+ @HiveDecimalVersionV2
+ public static HiveDecimal create(String string, boolean trimBlanks) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromString(
+ string, trimBlanks)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Create a HiveDecimal by parsing the characters in a whole byte array.
+ *
+ * Same rules as create(String string) above.
+ *
+ */
+ @HiveDecimalVersionV2
+ public static HiveDecimal create(byte[] bytes) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromBytes(
+ bytes, 0, bytes.length, false)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Same as the method above, except blanks before and after are tolerated.
+ *
+ */
+ @HiveDecimalVersionV2
+ public static HiveDecimal create(byte[] bytes, boolean trimBlanks) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromBytes(
+ bytes, 0, bytes.length, trimBlanks)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * This method takes in digits only UTF-8 characters, a sign flag, and a scale and returns
+ * a decimal.
+ */
+ @HiveDecimalVersionV2
+ public static HiveDecimal create(boolean isNegative, byte[] bytes, int scale) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromDigitsOnlyBytesAndScale(
+ isNegative, bytes, 0, bytes.length, scale)) {
+ return null;
+ }
+ if (isNegative) {
+ result.fastNegate();
+ }
+ return result;
+ }
+
+ @HiveDecimalVersionV2
+ public static HiveDecimal create(
+ boolean isNegative, byte[] bytes, int offset, int length, int scale) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromDigitsOnlyBytesAndScale(
+ isNegative, bytes, offset, length, scale)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Create a HiveDecimal by parsing the characters in a slice of a byte array.
+ *
+ * Same rules as create(String string) above.
+ *
+ */
+ @HiveDecimalVersionV2
+ public static HiveDecimal create(byte[] bytes, int offset, int length) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromBytes(
+ bytes, offset, length, false)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Same as the method above, except blanks before and after are tolerated.
+ *
+ */
+ @HiveDecimalVersionV2
+ public static HiveDecimal create(
+ byte[] bytes, int offset, int length, boolean trimBlanks) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromBytes(
+ bytes, offset, length, trimBlanks)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Create a HiveDecimal object from an int.
+ *
+ */
+ @HiveDecimalVersionV1
+ public static HiveDecimal create(int intValue) {
+ HiveDecimal result = new HiveDecimal();
+ result.fastSetFromInt(intValue);
+ return result;
+ }
+
+ /**
+ * Create a HiveDecimal object from a long.
+ *
+ */
+ @HiveDecimalVersionV1
+ public static HiveDecimal create(long longValue) {
+ HiveDecimal result = new HiveDecimal();
+ result.fastSetFromLong(longValue);
+ return result;
+ }
+
+ /**
+ * Create a HiveDecimal object from a long with a specified scale.
+ *
+ */
+ @HiveDecimalVersionV2
+ public static HiveDecimal create(long longValue, int scale) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromLongAndScale(
+ longValue, scale)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Create a HiveDecimal object from a float.
+ *
+ * This method is equivalent to HiveDecimal.create(Float.toString(floatValue))
+ */
+ @HiveDecimalVersionV2
+ public static HiveDecimal create(float floatValue) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromFloat(floatValue)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Create a HiveDecimal object from a double.
+ *
+ * This method is equivalent to HiveDecimal.create(Double.toString(doubleValue))
+ */
+ @HiveDecimalVersionV2
+ public static HiveDecimal create(double doubleValue) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromDouble(doubleValue)) {
+ return null;
+ }
+ return result;
+ }
+
+ //-----------------------------------------------------------------------------------------------
+ // Serialization methods.
+ //-----------------------------------------------------------------------------------------------
+
+ // The byte length of the scratch byte array that needs to be passed to serializationUtilsRead.
+ @HiveDecimalVersionV2
+ public static final int SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ =
+ FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ;
+
+ /**
+ * Deserialize data written in the format used by the SerializationUtils methods
+ * readBigInteger/writeBigInteger and create a decimal using the supplied scale.
+ *
+ * ORC uses those SerializationUtils methods for its serialization.
+ *
+ * A scratch bytes array is necessary to do the binary to decimal conversion for better
+ * performance. Pass a SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ byte array for scratchBytes.
+ *
+ * @return The deserialized decimal or null if the conversion failed.
+ */
+ @HiveDecimalVersionV2
+ public static HiveDecimal serializationUtilsRead(
+ InputStream inputStream, int scale,
+ byte[] scratchBytes)
+ throws IOException {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSerializationUtilsRead(
+ inputStream, scale,
+ scratchBytes)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Convert bytes in the format used by BigInteger's toByteArray format (and accepted by its
+ * constructor) into a decimal using the specified scale.
+ *
+ * Our bigIntegerBytes methods create bytes in this format, too.
+ *
+ * This method is designed for high performance and does not create an actual BigInteger during
+ * binary to decimal conversion.
+ *
+ */
+ @HiveDecimalVersionV2
+ public static HiveDecimal createFromBigIntegerBytesAndScale(
+ byte[] bytes, int scale) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromBigIntegerBytesAndScale(
+ bytes, 0, bytes.length, scale)) {
+ return null;
+ }
+ return result;
+ }
+
+ @HiveDecimalVersionV2
+ public static HiveDecimal createFromBigIntegerBytesAndScale(
+ byte[] bytes, int offset, int length, int scale) {
+ HiveDecimal result = new HiveDecimal();
+ if (!result.fastSetFromBigIntegerBytesAndScale(
+ bytes, offset, length, scale)) {
+ return null;
+ }
+ return result;
+ }
+
+ // The length of the long array that needs to be passed to serializationUtilsWrite.
+ @HiveDecimalVersionV2
+ public static final int SCRATCH_LONGS_LEN = FAST_SCRATCH_LONGS_LEN;
+
+ /**
+ * Serialize this decimal's BigInteger equivalent unscaled value using the format that the
+ * SerializationUtils methods readBigInteger/writeBigInteger use.
+ *
+ * ORC uses those SerializationUtils methods for its serialization.
+ *
+ * Scratch objects necessary to do the decimal to binary conversion without actually creating a
+ * BigInteger object are passed for better performance.
+ *
+ * Allocate scratchLongs with SCRATCH_LONGS_LEN longs.
+ *
+ */
+ @HiveDecimalVersionV2
+ public boolean serializationUtilsWrite(
+ OutputStream outputStream,
+ long[] scratchLongs)
+ throws IOException {
+ return
+ fastSerializationUtilsWrite(
+ outputStream,
+ scratchLongs);
+ }
+
+ // The length of the scratch byte array that needs to be passed to bigIntegerBytes, etc.
+ @HiveDecimalVersionV2
+ public static final int SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES =
+ FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES;
+
+ /**
+ * Return binary representation of this decimal's BigInteger equivalent unscaled value using
+ * the format that the BigInteger's toByteArray method returns (and the BigInteger constructor
+ * accepts).
+ *
+ * Used by LazyBinary, Avro, and Parquet serialization.
+ *
+ * Scratch objects necessary to do the decimal to binary conversion without actually creating a
+ * BigInteger object are passed for better performance.
+ *
+ * Allocate scratchLongs with SCRATCH_LONGS_LEN longs.
+ * And, allocate buffer with SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes.
+ *
+ * @param scratchLongs
+ * @param buffer
+ * @return The number of bytes used for the binary result in buffer. Otherwise, 0 if the
+ * conversion failed.
+ */
+ @HiveDecimalVersionV2
+ public int bigIntegerBytes(
+ long[] scratchLongs, byte[] buffer) {
+ return
+ fastBigIntegerBytes(
+ scratchLongs, buffer);
+ }
+
+ @HiveDecimalVersionV2
+ public byte[] bigIntegerBytes() {
+ long[] scratchLongs = new long[SCRATCH_LONGS_LEN];
+ byte[] buffer = new byte[SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES];
+ final int byteLength =
+ fastBigIntegerBytes(
+ scratchLongs, buffer);
+ return Arrays.copyOfRange(buffer, 0, byteLength);
+ }
+
+ /**
+ * Convert decimal to BigInteger binary bytes with a serialize scale, similar to the formatScale
+ * for toFormatString. It adds trailing zeroes the (emulated) BigInteger toByteArray result
+ * when a serializeScale is greater than current scale. Or, rounds if scale is less than
+ * current scale.
+ *
+ * Used by Avro and Parquet serialization.
+ *
+ * This emulates the OldHiveDecimal setScale AND THEN OldHiveDecimal getInternalStorage() behavior.
+ *
+ */
+ @HiveDecimalVersionV2
+ public int bigIntegerBytesScaled(
+ int serializeScale,
+ long[] scratchLongs, byte[] buffer) {
+ return
+ fastBigIntegerBytesScaled(
+ serializeScale,
+ scratchLongs, buffer);
+ }
+
+ @HiveDecimalVersionV2
+ public byte[] bigIntegerBytesScaled(int serializeScale) {
+ long[] scratchLongs = new long[SCRATCH_LONGS_LEN];
+ byte[] buffer = new byte[SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES];
+ int byteLength =
+ fastBigIntegerBytesScaled(
+ serializeScale,
+ scratchLongs, buffer);
+ return Arrays.copyOfRange(buffer, 0, byteLength);
+ }
+
+ //-----------------------------------------------------------------------------------------------
+ // Convert to string/UTF-8 ASCII bytes methods.
+ //-----------------------------------------------------------------------------------------------
+
+ /**
+ * Return a string representation of the decimal.
+ *
+ * It is the equivalent of calling bigDecimalValue().toPlainString -- it does not add exponent
+ * notation -- but is much faster.
+ *
+ * NOTE: If setScale(int serializationScale) was used to create the decimal object, then trailing
+ * fractional digits will be added to display to the serializationScale. Or, the display may
+ * get rounded. See the comments for that method.
+ *
+ */
+ @HiveDecimalVersionV1
+ @Override
+ public String toString() {
+ if (fastSerializationScale() != -1) {
+
+ // Use the serialization scale and format the string with trailing zeroes (or
+ // round the decimal) if necessary.
+ return
+ fastToFormatString(fastSerializationScale());
+ } else {
+ return
+ fastToString();
+ }
+ }
+
+ @HiveDecimalVersionV2
+ public String toString(
+ byte[] scratchBuffer) {
+ if (fastSerializationScale() != -1) {
+
+ // Use the serialization scale and format the string with trailing zeroes (or
+ // round the decimal) if necessary.
+ return
+ fastToFormatString(
+ fastSerializationScale(),
+ scratchBuffer);
+ } else {
+ return
+ fastToString(scratchBuffer);
+ }
+ }
+
+ /**
+ * Return a string representation of the decimal using the specified scale.
+ *
+ * This method is designed to ALWAYS SUCCEED (unless the newScale parameter is out of range).
+ *
+ * Is does the equivalent of a setScale(int newScale). So, more than 38 digits may be returned.
+ * See that method for more details on how this can happen.
+ *
+ * @param formatScale The number of digits after the decimal point
+ * @return The scaled decimal representation string representation.
+ */
+ @HiveDecimalVersionV1
+ public String toFormatString(int formatScale) {
+ return
+ fastToFormatString(
+ formatScale);
+ }
+
+ @HiveDecimalVersionV2
+ public String toFormatString(int formatScale, byte[] scratchBuffer) {
+ return
+ fastToFormatString(
+ formatScale,
+ scratchBuffer);
+ }
+
+ @HiveDecimalVersionV2
+ public String toDigitsOnlyString() {
+ return
+ fastToDigitsOnlyString();
+ }
+
+ // The length of the scratch buffer that needs to be passed to toBytes, toFormatBytes,
+ // toDigitsOnlyBytes.
+ @HiveDecimalVersionV2
+ public final static int SCRATCH_BUFFER_LEN_TO_BYTES = FAST_SCRATCH_BUFFER_LEN_TO_BYTES;
+
+ /**
+ * Decimal to ASCII bytes conversion.
+ *
+ * The scratch buffer will contain the result afterwards. It should be
+ * SCRATCH_BUFFER_LEN_TO_BYTES bytes long.
+ *
+ * The result is produced at the end of the scratch buffer, so the return value is the byte
+ * index of the first byte. The byte slice is [byteIndex:SCRATCH_BUFFER_LEN_TO_BYTES-1].
+ *
+ */
+ @HiveDecimalVersionV2
+ public int toBytes(
+ byte[] scratchBuffer) {
+ return
+ fastToBytes(
+ scratchBuffer);
+ }
+
+ /**
+ * This is the serialization version of decimal to string conversion.
+ *
+ * It adds trailing zeroes when the formatScale is greater than the current scale. Or, it
+ * does round if the formatScale is less than the current scale.
+ *
+ * Note that you can get more than 38 (MAX_PRECISION) digits in the output with this method.
+ *
+ */
+ @HiveDecimalVersionV2
+ public int toFormatBytes(
+ int formatScale,
+ byte[] scratchBuffer) {
+ return
+ fastToFormatBytes(
+ formatScale,
+ scratchBuffer);
+ }
+
+ /**
+ * Convert decimal to just the digits -- no dot.
+ *
+ * Currently used by BinarySortable serialization.
+ *
+ * A faster way to get just the digits than calling unscaledValue.toString().getBytes().
+ *
+ */
+ @HiveDecimalVersionV2
+ public int toDigitsOnlyBytes(
+ byte[] scratchBuffer) {
+ return
+ fastToDigitsOnlyBytes(
+ scratchBuffer);
+ }
+
+ //-----------------------------------------------------------------------------------------------
+ // Comparison methods.
+ //-----------------------------------------------------------------------------------------------
+
+ @HiveDecimalVersionV1
+ @Override
+ public int compareTo(HiveDecimal dec) {
+ return fastCompareTo(dec);
+ }
+
+ /**
+ * Hash code based on (new) decimal representation.
+ *
+ * Faster than hashCode().
+ *
+ * Used by map join and other Hive internal purposes where performance is important.
+ *
+ * IMPORTANT: See comments for hashCode(), too.
+ */
+ @HiveDecimalVersionV2
+ public int newFasterHashCode() {
+ return fastNewFasterHashCode();
+ }
+
+ /**
+ * This is returns original hash code as returned by HiveDecimalV1.
+ *
+ * We need this when the HiveDecimalV1 hash code has been exposed and and written or affected
+ * how data is written.
+ *
+ * NOTE: It is necessary to create a BigDecimal object and use its hash code, so this method is
+ * slow.
+ */
+ @HiveDecimalVersionV1
+ @Override
+ public int hashCode() {
+ return fastHashCode();
+ }
+
+ /**
+ * Are two decimal content (values) equal?
+ *
+ * @param obj The 2nd decimal.
+ * @return When obj is null or not class HiveDecimal, the return is false.
+ * Otherwise, returns true when the decimal values are exactly equal.
+ */
+ @HiveDecimalVersionV1
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || obj.getClass() != getClass()) {
+ return false;
+ }
+ return fastEquals((HiveDecimal) obj);
+ }
+
+
+ //-----------------------------------------------------------------------------------------------
+ // Attribute methods.
+ //-----------------------------------------------------------------------------------------------
+
+ /**
+ * Returns the scale of the decimal. Range 0 .. MAX_SCALE.
+ *
+ */
+ @HiveDecimalVersionV1
+ public int scale() {
+ return fastScale();
+ }
+
+ /**
+ * Returns the number of integer digits in the decimal.
+ *
+ * When the integer portion is zero, this method returns 0.
+ *
+ */
+ @HiveDecimalVersionV2
+ public int integerDigitCount() {
+ return fastIntegerDigitCount();
+ }
+
+ /**
+ * Returns the number of digits (integer and fractional) in the number, which is equivalent
+ * to SQL decimal precision.
+ *
+ * Note that this method is different from rawPrecision(), which returns the number of digits
+ * ignoring the scale. Note that rawPrecision returns 0 when the value is 0.
+ *
+ * Decimal precision rawPrecision
+ * 0 1 0
+ * 1 1 1
+ * -7 1 1
+ * 0.1 1 1
+ * 0.04 2 1
+ * 0.00380 5 3
+ * 104.0009 7 7
+ *
+ * If you just want the actual number of digits, use rawPrecision().
+ *
+ */
+ @HiveDecimalVersionV1
+ public int precision() {
+ return fastSqlPrecision();
+ }
+
+ // See comments for sqlPrecision.
+ @HiveDecimalVersionV2
+ public int rawPrecision() {
+ return fastRawPrecision();
+ }
+
+ /**
+ * Get the sign of the decimal.
+ *
+ * @return 0 if the decimal is equal to 0, -1 if less than zero, and 1 if greater than 0
+ */
+ @HiveDecimalVersionV1
+ public int signum() {
+ return fastSignum();
+ }
+
+ //-----------------------------------------------------------------------------------------------
+ // Value conversion methods.
+ //-----------------------------------------------------------------------------------------------
+
+ /**
+ * Is the decimal value a byte? Range -128 to 127.
+ * Byte.MIN_VALUE Byte.MAX_VALUE
+ *
+ * Emulates testing for no value corruption:
+ * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().byteValue()))
+ *
+ * NOTE: Fractional digits are ignored in the test since byteValue() will
+ * remove them (round down).
+ *
+ * @return True when byteValue() will return a correct byte.
+ */
+ @HiveDecimalVersionV2
+ public boolean isByte() {
+ return fastIsByte();
+ }
+
+ /**
+ * A byte variation of longValue()
+ *
+ * This method will return a corrupted value unless isByte() is true.
+ */
+ @HiveDecimalVersionV1
+ public byte byteValue() {
+ return fastByteValueClip();
+ }
+
+ /**
+ * Is the decimal value a short? Range -32,768 to 32,767.
+ * Short.MIN_VALUE Short.MAX_VALUE
+ *
+ * Emulates testing for no value corruption:
+ * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().shortValue()))
+ *
+ * NOTE: Fractional digits are ignored in the test since shortValue() will
+ * remove them (round down).
+ *
+ * @return True when shortValue() will return a correct short.
+ */
+ @HiveDecimalVersionV2
+ public boolean isShort() {
+ return fastIsShort();
+ }
+
+ /**
+ * A short variation of longValue().
+ *
+ * This method will return a corrupted value unless isShort() is true.
+ */
+ @HiveDecimalVersionV1
+ public short shortValue() {
+ return fastShortValueClip();
+ }
+
+ /**
+ * Is the decimal value a int? Range -2,147,483,648 to 2,147,483,647.
+ * Integer.MIN_VALUE Integer.MAX_VALUE
+ *
+ * Emulates testing for no value corruption:
+ * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().intValue()))
+ *
+ * NOTE: Fractional digits are ignored in the test since intValue() will
+ * remove them (round down).
+ *
+ * @return True when intValue() will return a correct int.
+ */
+ @HiveDecimalVersionV2
+ public boolean isInt() {
+ return fastIsInt();
+ }
+
+ /**
+ * An int variation of longValue().
+ *
+ * This method will return a corrupted value unless isInt() is true.
+ */
+ @HiveDecimalVersionV1
+ public int intValue() {
+ return fastIntValueClip();
+ }
+
+ /**
+ * Is the decimal value a long? Range -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807.
+ * Long.MIN_VALUE Long.MAX_VALUE
+ *
+ * Emulates testing for no value corruption:
+ * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().longValue()))
+ *
+ * NOTE: Fractional digits are ignored in the test since longValue() will
+ * remove them (round down).
+ *
+ * @return True when longValue() will return a correct long.
+ */
+ @HiveDecimalVersionV2
+ public boolean isLong() {
+ return fastIsLong();
+ }
+
+ /**
+ * Return the long value of a decimal.
+ *
+ * This method will return a corrupted value unless isLong() is true.
+ */
+ @HiveDecimalVersionV1
+ public long longValue() {
+ return fastLongValueClip();
+ }
+
+ @HiveDecimalVersionV1
+ public long longValueExact() {
+ if (!isLong()) {
+ throw new ArithmeticException();
+ }
+ return fastLongValueClip();
+ }
+
+ /**
+ * Return a float representing the decimal. Due the limitations of float, some values will not
+ * be accurate.
+ *
+ */
+ @HiveDecimalVersionV1
+ public float floatValue() {
+ return fastFloatValue();
+ }
+
+ /**
+ * Return a double representing the decimal. Due the limitations of double, some values will not
+ * be accurate.
+ *
+ */
+ @HiveDecimalVersionV1
+ public double doubleValue() {
+ return fastDoubleValue();
+ }
+
+ /**
+ * Return a BigDecimal representing the decimal. The BigDecimal class is able to accurately
+ * represent the decimal.
+ *
+ * NOTE: We are not representing our decimal as BigDecimal now as OldHiveDecimal did, so this
+ * is now slower.
+ *
+ */
+ @HiveDecimalVersionV1
+ public BigDecimal bigDecimalValue() {
+ return fastBigDecimalValue();
+ }
+
+ /**
+ * Get a BigInteger representing the decimal's digits without a dot.
+ *
+ * @return Returns a signed BigInteger.
+ */
+ @HiveDecimalVersionV1
+ public BigInteger unscaledValue() {
+ return fastBigIntegerValue();
+ }
+
+ /**
+ * Return a decimal with only the fractional digits.
+ *
+ * Zero is returned when there are no fractional digits (i.e. scale is 0).
+ *
+ */
+ @HiveDecimalVersionV2
+ public HiveDecimal fractionPortion() {
+ HiveDecimal result = new HiveDecimal();
+ result.fastFractionPortion();
+ return result;
+ }
+
+ /**
+ * Return a decimal with only the integer digits.
+ *
+ * Any fractional digits are removed. E.g. 2.083 scale 3 returns as 2 scale 0.
+ *
+ */
+ @HiveDecimalVersionV2
+ public HiveDecimal integerPortion() {
+ HiveDecimal result = new HiveDecimal();
+ result.fastIntegerPortion();
+ return result;
+ }
+
+ //-----------------------------------------------------------------------------------------------
+ // Math methods.
+ //-----------------------------------------------------------------------------------------------
+
+ /**
+ * Add the current decimal and another decimal and return the result.
+ *
+ */
+ @HiveDecimalVersionV1
+ public HiveDecimal add(HiveDecimal dec) {
+ HiveDecimal result = new HiveDecimal();
+ if (!fastAdd(
+ dec,
+ result)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Subtract from the current decimal another decimal and return the result.
+ *
+ */
+ @HiveDecimalVersionV1
+ public HiveDecimal subtract(HiveDecimal dec) {
+ HiveDecimal result = new HiveDecimal();
+ if (!fastSubtract(
+ dec,
+ result)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Multiply two decimals.
+ *
+ * NOTE: Overflow Determination for Multiply
+ *
+ * OldDecimal.multiply performs the multiply with BigDecimal but DOES NOT ALLOW ROUNDING
+ * (i.e. no throwing away lower fractional digits).
+ *
+ * CONSIDER: Allowing rounding. This would eliminate cases today where we return null for
+ * the multiplication result.
+ *
+ * IMPLEMENTATION NOTE: HiveDecimalV1 code does this:
+ *
+ * return create(bd.multiply(dec.bd), false);
+ */
+ @HiveDecimalVersionV1
+ public HiveDecimal multiply(HiveDecimal dec) {
+ HiveDecimal result = new HiveDecimal();
+ if (!fastMultiply(
+ dec,
+ result)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Multiplies a decimal by a power of 10.
+ *
+ * The decimal 19350 scale 0 will return 193.5 scale 1 when power is -2 (negative).
+ *
+ * The decimal 1.000923 scale 6 will return 10009.23 scale 2 when power is 4 (positive).
+ *
+ * @param power
+ * @return Returns a HiveDecimal whose value is value * 10^power.
+ */
+ @HiveDecimalVersionV1
+ public HiveDecimal scaleByPowerOfTen(int power) {
+ if (power == 0 || fastSignum() == 0) {
+ // No change for multiply by 10^0 or value 0.
+ return this;
+ }
+ HiveDecimal result = new HiveDecimal();
+ if (!fastScaleByPowerOfTen(
+ power,
+ result)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Take the absolute value of a decimal.
+ *
+ * @return When the decimal is negative, returns a new HiveDecimal with the positive value.
+ * Otherwise, returns the current 0 or positive value object;
+ */
+ @HiveDecimalVersionV1
+ public HiveDecimal abs() {
+ if (fastSignum() != -1) {
+ return this;
+ }
+ HiveDecimal result = new HiveDecimal(this);
+ result.fastAbs();
+ return result;
+ }
+
+ /**
+ * Reverse the sign of a decimal.
+ *
+ * @return Returns a new decimal with the sign flipped. When the value is 0, the current
+ * object is returned.
+ */
+ @HiveDecimalVersionV1
+ public HiveDecimal negate() {
+ if (fastSignum() == 0) {
+ return this;
+ }
+ HiveDecimal result = new HiveDecimal(this);
+ result.fastNegate();
+ return result;
+ }
+
+ //-----------------------------------------------------------------------------------------------
+ // Rounding / setScale methods.
+ //-----------------------------------------------------------------------------------------------
+
+ /**
+ * DEPRECATED for V2.
+ *
+ * Create a decimal from another decimal whose only change is it is MARKED and will display /
+ * serialize with a specified scale that will add trailing zeroes (or round) if necessary.
+ *
+ * After display / serialization, the MARKED object is typically thrown away.
+ *
+ * A MARKED decimal ONLY affects these 2 methods since these were the only ways setScale was
+ * used in the old code.
+ *
+ * toString
+ * unscaleValue
+ *
+ * This method has been deprecated because has poor performance by creating a throw away object.
+ *
+ * For setScale(scale).toString() use toFormatString(scale) instead.
+ * For setScale(scale).unscaledValue().toByteArray() use V2 bigIntegerBytesScaled(scale) instead.
+ *
+ * For better performance, use the V2 form of toFormatString that takes a scratch buffer,
+ * or even better use toFormatBytes.
+ *
+ * And, use the form of bigIntegerBytesScaled that takes scratch objects for better performance.
+ *
+ */
+ @Deprecated
+ @HiveDecimalVersionV1
+ public HiveDecimal setScale(int serializationScale) {
+ HiveDecimal result = new HiveDecimal(this);
+ result.fastSetSerializationScale(serializationScale);
+ return result;
+ }
+
+ /**
+ * Do decimal rounding and return the result.
+ *
+ * When the roundingPoint is 0 or positive, we round away lower fractional digits if the
+ * roundingPoint is less than current scale. In this case, we will round the result using the
+ * specified rounding mode.
+ *
+ * When the roundingPoint is negative, the rounding will occur within the integer digits. Integer
+ * digits below the roundPoint will be cleared. If the rounding occurred, a one will be added
+ * just above the roundingPoint. Note this may cause overflow.
+ *
+ * No effect when the roundingPoint equals the current scale. The current object is returned.
+ *
+ * The name setScale is taken from BigDecimal.setScale -- a better name would have been round.
+ *
+ */
+ @HiveDecimalVersionV1
+ public HiveDecimal setScale(
+ int roundingPoint, int roundingMode) {
+ if (fastScale() == roundingPoint) {
+ // No change.
+ return this;
+ }
+
+ // Even if we are just setting the scale when newScale is greater than the current scale,
+ // we need a new object to obey our immutable behavior.
+ HiveDecimal result = new HiveDecimal();
+ if (!fastRound(
+ roundingPoint, roundingMode,
+ result)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Return the result of decimal^exponent
+ *
+ * CONSIDER: Currently, negative exponent is not supported.
+ * CONSIDER: Does anybody use this method?
+ *
+ */
+ @HiveDecimalVersionV1
+ public HiveDecimal pow(int exponent) {
+ HiveDecimal result = new HiveDecimal(this);
+ if (!fastPow(
+ exponent, result)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Divides this decimal by another decimal and returns a new decimal with the result.
+ *
+ */
+ @HiveDecimalVersionV1
+ public HiveDecimal divide(HiveDecimal divisor) {
+ HiveDecimal result = new HiveDecimal();
+ if (!fastDivide(
+ divisor,
+ result)) {
+ return null;
+ }
+ return result;
+ }
+
+ /**
+ * Divides this decimal by another decimal and returns a new decimal with the remainder of the
+ * division.
+ *
+ * value is (decimal % divisor)
+ *
+ * The remainder is equivalent to BigDecimal:
+ * bigDecimalValue().subtract(bigDecimalValue().divideToIntegralValue(divisor).multiply(divisor))
+ *
+ */
+ @HiveDecimalVersionV1
+ public HiveDecimal remainder(HiveDecimal divisor) {
+ HiveDecimal result = new HiveDecimal();
+ if (!fastRemainder(
+ divisor,
+ result)) {
+ return null;
+ }
+ return result;
+ }
+
+ //-----------------------------------------------------------------------------------------------
+ // Precision/scale enforcement methods.
+ //-----------------------------------------------------------------------------------------------
+
+ /**
+ * Determine if a decimal fits within a specified maxPrecision and maxScale, and round
+ * off fractional digits if necessary to make the decimal fit.
+ *
+ * The relationship between the enforcement maxPrecision and maxScale is restricted. The
+ * specified maxScale must be less than or equal to the maxPrecision.
+ *
+ * Normally, decimals that result from creation operation, arithmetic operations, etc are
+ * "free range" up to MAX_PRECISION and MAX_SCALE. Each operation checks if the result decimal
+ * is beyond MAX_PRECISION and MAX_SCALE. If so the result decimal is rounded off using
+ * ROUND_HALF_UP. If the round digit is 5 or more, one is added to the lowest remaining digit.
+ * The round digit is the digit just below the round point. Result overflow can occur if a
+ * result decimal's integer portion exceeds MAX_PRECISION.
+ *
+ * This method supports enforcing to a declared Hive DECIMAL's precision/scale.
+ * E.g. DECIMAL(10,4)
+ *
+ * Here are the enforcement/rounding checks of this method:
+ *
+ * If the decimal's integer digit count exceeds this, the decimal does not fit (overflow).
+ *
+ * 2) If decimal's scale is greater than maxScale, then excess fractional digits are
+ * rounded off. When rounding increases the remaining decimal, it may exceed the
+ * limits and overflow.
+ *
+ * @param dec
+ * @param maxPrecision
+ * @param maxScale
+ * @return The original decimal if no adjustment is necessary.
+ * A rounded off decimal if adjustment was necessary.
+ * Otherwise, null if the decimal doesn't fit within maxPrecision / maxScale or rounding
+ * caused a result that exceeds the specified limits or MAX_PRECISION integer digits.
+ */
+ @HiveDecimalVersionV1
+ public static HiveDecimal enforcePrecisionScale(
+ HiveDecimal dec, int maxPrecision, int maxScale) {
+
+ if (maxPrecision < 1 || maxPrecision > MAX_PRECISION) {
+ throw new IllegalArgumentException(STRING_ENFORCE_PRECISION_OUT_OF_RANGE);
+ }
+
+ if (maxScale < 0 || maxScale > HiveDecimal.MAX_SCALE) {
+ throw new IllegalArgumentException(STRING_ENFORCE_SCALE_OUT_OF_RANGE);
+ }
+
+ if (maxPrecision < maxScale) {
+ throw new IllegalArgumentException(STRING_ENFORCE_SCALE_LESS_THAN_EQUAL_PRECISION);
+ }
+
+ if (dec == null) {
+ return null;
+ }
+
+ FastCheckPrecisionScaleStatus status =
+ dec.fastCheckPrecisionScale(
+ maxPrecision, maxScale);
+ switch (status) {
+ case NO_CHANGE:
+ return dec;
+ case OVERFLOW:
+ return null;
+ case UPDATE_SCALE_DOWN:
+ {
+ HiveDecimal result = new HiveDecimal();
+ if (!dec.fastUpdatePrecisionScale(
+ maxPrecision, maxScale, status,
+ result)) {
+ return null;
+ }
+ return result;
+ }
+ default:
+ throw new RuntimeException("Unknown fast decimal check precision and scale status " + status);
+ }
+ }
+
+ //-----------------------------------------------------------------------------------------------
+ // Validation methods.
+ //-----------------------------------------------------------------------------------------------
+
+ /**
+ * Throws an exception if the current decimal value is invalid.
+ */
+ @HiveDecimalVersionV2
+ public void validate() {
+ if (!fastIsValid()) {
+ fastRaiseInvalidException();
+ }
+ }
+}
\ No newline at end of file
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalV1.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalV1.java
new file mode 100644
index 0000000000000000000000000000000000000000..f99ffee32763ea3e61859e4c52cda9af85ed73a2
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalV1.java
@@ -0,0 +1,386 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.type;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.RoundingMode;
+
+/**
+ *
+ * HiveDecimal. Simple wrapper for BigDecimal. Adds fixed max precision and non scientific string
+ * representation
+ *
+ */
+public final class HiveDecimalV1 implements Comparable {
+ @HiveDecimalVersionV1
+ public static final int MAX_PRECISION = 38;
+ @HiveDecimalVersionV1
+ public static final int MAX_SCALE = 38;
+
+ /**
+ * Default precision/scale when user doesn't specify in the column metadata, such as
+ * decimal and decimal(8).
+ */
+ @HiveDecimalVersionV1
+ public static final int USER_DEFAULT_PRECISION = 10;
+ @HiveDecimalVersionV1
+ public static final int USER_DEFAULT_SCALE = 0;
+
+ /**
+ * Default precision/scale when system is not able to determine them, such as in case
+ * of a non-generic udf.
+ */
+ @HiveDecimalVersionV1
+ public static final int SYSTEM_DEFAULT_PRECISION = 38;
+ @HiveDecimalVersionV1
+ public static final int SYSTEM_DEFAULT_SCALE = 18;
+
+ @HiveDecimalVersionV1
+ public static final HiveDecimalV1 ZERO = new HiveDecimalV1(BigDecimal.ZERO);
+ @HiveDecimalVersionV1
+ public static final HiveDecimalV1 ONE = new HiveDecimalV1(BigDecimal.ONE);
+
+ @HiveDecimalVersionV1
+ public static final int ROUND_FLOOR = BigDecimal.ROUND_FLOOR;
+ @HiveDecimalVersionV1
+ public static final int ROUND_CEILING = BigDecimal.ROUND_CEILING;
+ @HiveDecimalVersionV1
+ public static final int ROUND_HALF_UP = BigDecimal.ROUND_HALF_UP;
+ @HiveDecimalVersionV1
+ public static final int ROUND_HALF_EVEN = BigDecimal.ROUND_HALF_EVEN;
+
+ private BigDecimal bd = BigDecimal.ZERO;
+
+ private HiveDecimalV1(BigDecimal bd) {
+ this.bd = bd;
+ }
+
+ @HiveDecimalVersionV1
+ public static HiveDecimalV1 create(BigDecimal b) {
+ return create(b, true);
+ }
+
+ @HiveDecimalVersionV1
+ public static HiveDecimalV1 create(BigDecimal b, boolean allowRounding) {
+ BigDecimal bd = normalize(b, allowRounding);
+ return bd == null ? null : new HiveDecimalV1(bd);
+ }
+
+ @HiveDecimalVersionV1
+ public static HiveDecimalV1 create(BigInteger unscaled, int scale) {
+ BigDecimal bd = normalize(new BigDecimal(unscaled, scale), true);
+ return bd == null ? null : new HiveDecimalV1(bd);
+ }
+
+ @HiveDecimalVersionV1
+ public static HiveDecimalV1 create(String dec) {
+ BigDecimal bd;
+ try {
+ bd = new BigDecimal(dec.trim());
+ } catch (NumberFormatException ex) {
+ return null;
+ }
+ bd = normalize(bd, true);
+ return bd == null ? null : new HiveDecimalV1(bd);
+ }
+
+ @HiveDecimalVersionV1
+ public static HiveDecimalV1 create(BigInteger bi) {
+ BigDecimal bd = normalize(new BigDecimal(bi), true);
+ return bd == null ? null : new HiveDecimalV1(bd);
+ }
+
+ @HiveDecimalVersionV1
+ public static HiveDecimalV1 create(int i) {
+ return new HiveDecimalV1(new BigDecimal(i));
+ }
+
+ @HiveDecimalVersionV1
+ public static HiveDecimalV1 create(long l) {
+ return new HiveDecimalV1(new BigDecimal(l));
+ }
+
+ @HiveDecimalVersionV1
+ @Override
+ public String toString() {
+ return bd.toPlainString();
+ }
+
+ /**
+ * Return a string representation of the number with the number of decimal digits as
+ * the given scale. Please note that this is different from toString().
+ * @param scale the number of digits after the decimal point
+ * @return the string representation of exact number of decimal digits
+ */
+ @HiveDecimalVersionV1
+ public String toFormatString(int scale) {
+ return (bd.scale() == scale ? bd :
+ bd.setScale(scale, RoundingMode.HALF_UP)).toPlainString();
+ }
+
+ @HiveDecimalVersionV1
+ public HiveDecimalV1 setScale(int i) {
+ return new HiveDecimalV1(bd.setScale(i, RoundingMode.HALF_UP));
+ }
+
+ @HiveDecimalVersionV1
+ @Override
+ public int compareTo(HiveDecimalV1 dec) {
+ return bd.compareTo(dec.bd);
+ }
+
+ @HiveDecimalVersionV1
+ @Override
+ public int hashCode() {
+ return bd.hashCode();
+ }
+
+ @HiveDecimalVersionV1
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || obj.getClass() != getClass()) {
+ return false;
+ }
+ return bd.equals(((HiveDecimalV1) obj).bd);
+ }
+
+ @HiveDecimalVersionV1
+ public int scale() {
+ return bd.scale();
+ }
+
+ /**
+ * Returns the number of digits (integer and fractional) in the number, which is equivalent
+ * to SQL decimal precision. Note that this is different from BigDecimal.precision(),
+ * which returns the precision of the unscaled value (BigDecimal.valueOf(0.01).precision() = 1,
+ * whereas HiveDecimal.create("0.01").precision() = 2).
+ * If you want the BigDecimal precision, use HiveDecimal.bigDecimalValue().precision()
+ * @return
+ */
+ @HiveDecimalVersionV1
+ public int precision() {
+ int bdPrecision = bd.precision();
+ int bdScale = bd.scale();
+
+ if (bdPrecision < bdScale) {
+ // This can happen for numbers less than 0.1
+ // For 0.001234: bdPrecision=4, bdScale=6
+ // In this case, we'll set the type to have the same precision as the scale.
+ return bdScale;
+ }
+ return bdPrecision;
+ }
+
+ /** Note - this method will corrupt the value if it doesn't fit. */
+ @HiveDecimalVersionV1
+ public int intValue() {
+ return bd.intValue();
+ }
+
+ @HiveDecimalVersionV1
+ public double doubleValue() {
+ return bd.doubleValue();
+ }
+
+ /** Note - this method will corrupt the value if it doesn't fit. */
+ @HiveDecimalVersionV1
+ public long longValue() {
+ return bd.longValue();
+ }
+
+ /** Note - this method will corrupt the value if it doesn't fit. */
+ @HiveDecimalVersionV1
+ public short shortValue() {
+ return bd.shortValue();
+ }
+
+ @HiveDecimalVersionV1
+ public float floatValue() {
+ return bd.floatValue();
+ }
+
+ @HiveDecimalVersionV1
+ public BigDecimal bigDecimalValue() {
+ return bd;
+ }
+
+ @HiveDecimalVersionV1
+ public byte byteValue() {
+ return bd.byteValue();
+ }
+
+ @HiveDecimalVersionV1
+ public HiveDecimalV1 setScale(int adjustedScale, int rm) {
+ return create(bd.setScale(adjustedScale, rm));
+ }
+
+ @HiveDecimalVersionV1
+ public HiveDecimalV1 subtract(HiveDecimalV1 dec) {
+ return create(bd.subtract(dec.bd));
+ }
+
+ @HiveDecimalVersionV1
+ public HiveDecimalV1 multiply(HiveDecimalV1 dec) {
+ return create(bd.multiply(dec.bd), false);
+ }
+
+ @HiveDecimalVersionV1
+ public BigInteger unscaledValue() {
+ return bd.unscaledValue();
+ }
+
+ @HiveDecimalVersionV1
+ public HiveDecimalV1 scaleByPowerOfTen(int n) {
+ return create(bd.scaleByPowerOfTen(n));
+ }
+
+ @HiveDecimalVersionV1
+ public HiveDecimalV1 abs() {
+ return create(bd.abs());
+ }
+
+ @HiveDecimalVersionV1
+ public HiveDecimalV1 negate() {
+ return create(bd.negate());
+ }
+
+ @HiveDecimalVersionV1
+ public HiveDecimalV1 add(HiveDecimalV1 dec) {
+ return create(bd.add(dec.bd));
+ }
+
+ @HiveDecimalVersionV1
+ public HiveDecimalV1 pow(int n) {
+ BigDecimal result = normalize(bd.pow(n), false);
+ return result == null ? null : new HiveDecimalV1(result);
+ }
+
+ @HiveDecimalVersionV1
+ public HiveDecimalV1 remainder(HiveDecimalV1 dec) {
+ return create(bd.remainder(dec.bd));
+ }
+
+ @HiveDecimalVersionV1
+ public HiveDecimalV1 divide(HiveDecimalV1 dec) {
+ return create(bd.divide(dec.bd, MAX_SCALE, RoundingMode.HALF_UP), true);
+ }
+
+ /**
+ * Get the sign of the underlying decimal.
+ * @return 0 if the decimal is equal to 0, -1 if less than zero, and 1 if greater than 0
+ */
+ @HiveDecimalVersionV1
+ public int signum() {
+ return bd.signum();
+ }
+
+ private static BigDecimal trim(BigDecimal d) {
+ if (d.compareTo(BigDecimal.ZERO) == 0) {
+ // Special case for 0, because java doesn't strip zeros correctly on that number.
+ d = BigDecimal.ZERO;
+ } else {
+ d = d.stripTrailingZeros();
+ if (d.scale() < 0) {
+ // no negative scale decimals
+ d = d.setScale(0);
+ }
+ }
+ return d;
+ }
+
+ private static BigDecimal normalize(BigDecimal bd, boolean allowRounding) {
+ if (bd == null) {
+ return null;
+ }
+
+ bd = trim(bd);
+
+ int intDigits = bd.precision() - bd.scale();
+
+ if (intDigits > MAX_PRECISION) {
+ return null;
+ }
+
+ int maxScale = Math.min(MAX_SCALE, Math.min(MAX_PRECISION - intDigits, bd.scale()));
+ if (bd.scale() > maxScale ) {
+ if (allowRounding) {
+ bd = bd.setScale(maxScale, RoundingMode.HALF_UP);
+ // Trimming is again necessary, because rounding may introduce new trailing 0's.
+ bd = trim(bd);
+ } else {
+ bd = null;
+ }
+ }
+
+ return bd;
+ }
+
+ private static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, int maxScale) {
+ if (bd == null) {
+ return null;
+ }
+
+ /**
+ * Specially handling the case that bd=0, and we are converting it to a type where precision=scale,
+ * such as decimal(1, 1).
+ */
+ if (bd.compareTo(BigDecimal.ZERO) == 0 && bd.scale() == 0 && maxPrecision == maxScale) {
+ return bd.setScale(maxScale);
+ }
+
+ bd = trim(bd);
+
+ if (bd.scale() > maxScale) {
+ bd = bd.setScale(maxScale, RoundingMode.HALF_UP);
+ }
+
+ int maxIntDigits = maxPrecision - maxScale;
+ int intDigits = bd.precision() - bd.scale();
+ if (intDigits > maxIntDigits) {
+ return null;
+ }
+
+ return bd;
+ }
+
+ @HiveDecimalVersionV1
+ public static HiveDecimalV1 enforcePrecisionScale(HiveDecimalV1 dec, int maxPrecision, int maxScale) {
+ if (dec == null) {
+ return null;
+ }
+
+ // Minor optimization, avoiding creating new objects.
+ if (dec.precision() - dec.scale() <= maxPrecision - maxScale &&
+ dec.scale() <= maxScale) {
+ return dec;
+ }
+
+ BigDecimal bd = enforcePrecisionScale(dec.bd, maxPrecision, maxScale);
+ if (bd == null) {
+ return null;
+ }
+
+ return HiveDecimalV1.create(bd);
+ }
+
+ @HiveDecimalVersionV1
+ public long longValueExact() {
+ return bd.longValueExact();
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV1.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV1.java
new file mode 100644
index 0000000000000000000000000000000000000000..82b769a15eb9fa884f44894e8555b7c80dbdfc50
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV1.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.type;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/**
+ * Marks methods including static methods and fields as being part of version 1 HiveDecimal.
+ *
+ */
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+public @interface HiveDecimalVersionV1 {
+
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV2.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV2.java
new file mode 100644
index 0000000000000000000000000000000000000000..a47513ebacb9c16600e9e2e50c194d13c19d39c9
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimalVersionV2.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common.type;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/**
+ * Marks methods including static methods and fields as being part of version 2 HiveDecimal.
+ *
+ */
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+public @interface HiveDecimalVersionV2 {
+
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java
new file mode 100644
index 0000000000000000000000000000000000000000..cb1306ee78ca99c00ef1a2ab8fcf439029f58dd4
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.type;
+
+import java.math.BigDecimal;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.hive.common.util.IntervalDayTimeUtils;
+
+
+/**
+ * Day-time interval type representing an offset in days/hours/minutes/seconds,
+ * with nanosecond precision.
+ * 1 day = 24 hours = 1440 minutes = 86400 seconds
+ */
+public class HiveIntervalDayTime implements Comparable {
+
+ // days/hours/minutes/seconds all represented as seconds
+ protected long totalSeconds;
+ protected int nanos;
+
+ public HiveIntervalDayTime() {
+ }
+
+ public HiveIntervalDayTime(int days, int hours, int minutes, int seconds, int nanos) {
+ set(days, hours, minutes, seconds, nanos);
+ }
+
+ public HiveIntervalDayTime(long seconds, int nanos) {
+ set(seconds, nanos);
+ }
+
+ public HiveIntervalDayTime(BigDecimal seconds) {
+ set(seconds);
+ }
+
+ public HiveIntervalDayTime(HiveIntervalDayTime other) {
+ set(other.totalSeconds, other.nanos);
+ }
+
+ public int getDays() {
+ return (int) TimeUnit.SECONDS.toDays(totalSeconds);
+ }
+
+ public int getHours() {
+ return (int) (TimeUnit.SECONDS.toHours(totalSeconds) % TimeUnit.DAYS.toHours(1));
+ }
+
+ public int getMinutes() {
+ return (int) (TimeUnit.SECONDS.toMinutes(totalSeconds) % TimeUnit.HOURS.toMinutes(1));
+ }
+
+ public int getSeconds() {
+ return (int) (totalSeconds % TimeUnit.MINUTES.toSeconds(1));
+ }
+
+ public int getNanos() {
+ return nanos;
+ }
+
+ /**
+ * Returns days/hours/minutes all converted into seconds.
+ * Nanos still need to be retrieved using getNanos()
+ * @return
+ */
+ public long getTotalSeconds() {
+ return totalSeconds;
+ }
+
+ /**
+ *
+ * @return double representation of the interval day time, accurate to nanoseconds
+ */
+ public double getDouble() {
+ return totalSeconds + nanos / 1000000000;
+ }
+
+ /**
+ * Ensures that the seconds and nanoseconds fields have consistent sign
+ */
+ protected void normalizeSecondsAndNanos() {
+ if (totalSeconds > 0 && nanos < 0) {
+ --totalSeconds;
+ nanos += IntervalDayTimeUtils.NANOS_PER_SEC;
+ } else if (totalSeconds < 0 && nanos > 0) {
+ ++totalSeconds;
+ nanos -= IntervalDayTimeUtils.NANOS_PER_SEC;
+ }
+ }
+
+ public void set(int days, int hours, int minutes, int seconds, int nanos) {
+ long totalSeconds = seconds;
+ totalSeconds += TimeUnit.DAYS.toSeconds(days);
+ totalSeconds += TimeUnit.HOURS.toSeconds(hours);
+ totalSeconds += TimeUnit.MINUTES.toSeconds(minutes);
+ totalSeconds += TimeUnit.NANOSECONDS.toSeconds(nanos);
+ nanos = nanos % IntervalDayTimeUtils.NANOS_PER_SEC;
+
+ this.totalSeconds = totalSeconds;
+ this.nanos = nanos;
+
+ normalizeSecondsAndNanos();
+ }
+
+ public void set(long seconds, int nanos) {
+ this.totalSeconds = seconds;
+ this.nanos = nanos;
+ normalizeSecondsAndNanos();
+ }
+
+ public void set(BigDecimal totalSecondsBd) {
+ long totalSeconds = totalSecondsBd.longValue();
+ BigDecimal fractionalSecs = totalSecondsBd.remainder(BigDecimal.ONE);
+ int nanos = fractionalSecs.multiply(IntervalDayTimeUtils.NANOS_PER_SEC_BD).intValue();
+ set(totalSeconds, nanos);
+ }
+
+ public void set(HiveIntervalDayTime other) {
+ set(other.getTotalSeconds(), other.getNanos());
+ }
+
+ public HiveIntervalDayTime negate() {
+ return new HiveIntervalDayTime(-getTotalSeconds(), -getNanos());
+ }
+
+ @Override
+ public int compareTo(HiveIntervalDayTime other) {
+ long cmp = this.totalSeconds - other.totalSeconds;
+ if (cmp == 0) {
+ cmp = this.nanos - other.nanos;
+ }
+ if (cmp != 0) {
+ cmp = cmp > 0 ? 1 : -1;
+ }
+ return (int) cmp;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof HiveIntervalDayTime)) {
+ return false;
+ }
+ return 0 == compareTo((HiveIntervalDayTime) obj);
+ }
+
+ /**
+ * Return a copy of this object.
+ */
+ @Override
+ public Object clone() {
+ return new HiveIntervalDayTime(totalSeconds, nanos);
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder().append(totalSeconds).append(nanos).toHashCode();
+ }
+
+ @Override
+ public String toString() {
+ // If normalize() was used, then day-hour-minute-second-nanos should have the same sign.
+ // This is currently working with that assumption.
+ boolean isNegative = (totalSeconds < 0 || nanos < 0);
+ String daySecondSignStr = isNegative ? "-" : "";
+
+ return String.format("%s%d %02d:%02d:%02d.%09d",
+ daySecondSignStr, Math.abs(getDays()),
+ Math.abs(getHours()), Math.abs(getMinutes()),
+ Math.abs(getSeconds()), Math.abs(getNanos()));
+ }
+
+ public static HiveIntervalDayTime valueOf(String strVal) {
+ HiveIntervalDayTime result = null;
+ if (strVal == null) {
+ throw new IllegalArgumentException("Interval day-time string was null");
+ }
+ Matcher patternMatcher = PATTERN_MATCHER.get();
+ patternMatcher.reset(strVal);
+ if (patternMatcher.matches()) {
+ // Parse out the individual parts
+ try {
+ // Sign - whether interval is positive or negative
+ int sign = 1;
+ String field = patternMatcher.group(1);
+ if (field != null && field.equals("-")) {
+ sign = -1;
+ }
+ int days = sign *
+ IntervalDayTimeUtils.parseNumericValueWithRange("day", patternMatcher.group(2),
+ 0, Integer.MAX_VALUE);
+ byte hours = (byte) (sign *
+ IntervalDayTimeUtils.parseNumericValueWithRange("hour", patternMatcher.group(3), 0, 23));
+ byte minutes = (byte) (sign *
+ IntervalDayTimeUtils.parseNumericValueWithRange("minute", patternMatcher.group(4), 0, 59));
+ int seconds = 0;
+ int nanos = 0;
+ field = patternMatcher.group(5);
+ if (field != null) {
+ BigDecimal bdSeconds = new BigDecimal(field);
+ if (bdSeconds.compareTo(IntervalDayTimeUtils.MAX_INT_BD) > 0) {
+ throw new IllegalArgumentException("seconds value of " + bdSeconds + " too large");
+ }
+ seconds = sign * bdSeconds.intValue();
+ nanos = sign * bdSeconds.subtract(new BigDecimal(bdSeconds.toBigInteger()))
+ .multiply(IntervalDayTimeUtils.NANOS_PER_SEC_BD).intValue();
+ }
+
+ result = new HiveIntervalDayTime(days, hours, minutes, seconds, nanos);
+ } catch (Exception err) {
+ throw new IllegalArgumentException("Error parsing interval day-time string: " + strVal, err);
+ }
+ } else {
+ throw new IllegalArgumentException(
+ "Interval string does not match day-time format of 'd h:m:s.n': " + strVal);
+ }
+
+ return result;
+ }
+
+ // Simple pattern: D H:M:S.nnnnnnnnn
+ private final static String PARSE_PATTERN =
+ "([+|-])?(\\d+) (\\d+):(\\d+):((\\d+)(\\.(\\d+))?)";
+
+ private static final ThreadLocal PATTERN_MATCHER = new ThreadLocal() {
+ @Override
+ protected Matcher initialValue() {
+ return Pattern.compile(PARSE_PATTERN).matcher("");
+ }
+ };
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java
new file mode 100644
index 0000000000000000000000000000000000000000..8ff95e343d59d12bf70052d33bd58a8f5c2d4ef2
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.type;
+
+
+/**
+ *
+ * HiveVarChar.
+ * String wrapper to support SQL VARCHAR features.
+ * Max string length is enforced.
+ *
+ */
+public class HiveVarchar extends HiveBaseChar
+ implements Comparable {
+
+ public static final int MAX_VARCHAR_LENGTH = 65535;
+
+ public HiveVarchar() {
+ }
+
+ public HiveVarchar(String val, int len) {
+ setValue(val, len);
+ }
+
+ public HiveVarchar(HiveVarchar hc, int len) {
+ setValue(hc, len);
+ }
+
+ /**
+ * Set the new value
+ */
+ public void setValue(String val) {
+ super.setValue(val, -1);
+ }
+
+ public void setValue(HiveVarchar hc) {
+ super.setValue(hc.getValue(), -1);
+ }
+
+ public int compareTo(HiveVarchar rhs) {
+ if (rhs == this) {
+ return 0;
+ }
+ return this.getValue().compareTo(rhs.getValue());
+ }
+
+ public boolean equals(Object rhs) {
+ if (rhs == this) {
+ return true;
+ }
+ return this.getValue().equals(((HiveVarchar)rhs).getValue());
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java
new file mode 100644
index 0000000000000000000000000000000000000000..eeb3359b3f225ff0f701f6d8c6c8a6aae8ae522d
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.type;
+
+import java.sql.Date;
+import java.sql.Timestamp;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+public class RandomTypeUtil {
+
+ public static String getRandString(Random r) {
+ return getRandString(r, null, r.nextInt(10));
+ }
+
+ public static String getRandString(Random r, String characters, int length) {
+ if (characters == null) {
+ characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+
+ }
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < length; i++) {
+ if (characters == null) {
+ sb.append((char) (r.nextInt(128)));
+ } else {
+ sb.append(characters.charAt(r.nextInt(characters.length())));
+ }
+ }
+ return sb.toString();
+ }
+
+ public static byte[] getRandBinary(Random r, int len){
+ byte[] bytes = new byte[len];
+ for (int j = 0; j < len; j++){
+ bytes[j] = Byte.valueOf((byte) r.nextInt());
+ }
+ return bytes;
+ }
+
+ private static final String DECIMAL_CHARS = "0123456789";
+
+ public static HiveDecimal getRandHiveDecimal(Random r) {
+ int precision;
+ int scale;
+ while (true) {
+ StringBuilder sb = new StringBuilder();
+ precision = 1 + r.nextInt(18);
+ scale = 0 + r.nextInt(precision + 1);
+
+ int integerDigits = precision - scale;
+
+ if (r.nextBoolean()) {
+ sb.append("-");
+ }
+
+ if (integerDigits == 0) {
+ sb.append("0");
+ } else {
+ sb.append(getRandString(r, DECIMAL_CHARS, integerDigits));
+ }
+ if (scale != 0) {
+ sb.append(".");
+ sb.append(getRandString(r, DECIMAL_CHARS, scale));
+ }
+
+ return HiveDecimal.create(sb.toString());
+ }
+ }
+
+ public static Date getRandDate(Random r) {
+ String dateStr = String.format("%d-%02d-%02d",
+ Integer.valueOf(1800 + r.nextInt(500)), // year
+ Integer.valueOf(1 + r.nextInt(12)), // month
+ Integer.valueOf(1 + r.nextInt(28))); // day
+ Date dateVal = Date.valueOf(dateStr);
+ return dateVal;
+ }
+
+ /**
+ * TIMESTAMP.
+ */
+
+ public static final long NANOSECONDS_PER_SECOND = TimeUnit.SECONDS.toNanos(1);
+ public static final long MILLISECONDS_PER_SECOND = TimeUnit.SECONDS.toMillis(1);
+ public static final long NANOSECONDS_PER_MILLISSECOND = TimeUnit.MILLISECONDS.toNanos(1);
+
+ private static final ThreadLocal DATE_FORMAT =
+ new ThreadLocal() {
+ @Override
+ protected DateFormat initialValue() {
+ return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ }
+ };
+
+ // We've switched to Joda/Java Calendar which has a more limited time range....
+ public static final int MIN_YEAR = 1900;
+ public static final int MAX_YEAR = 3000;
+ private static final long MIN_FOUR_DIGIT_YEAR_MILLIS = parseToMillis("1900-01-01 00:00:00");
+ private static final long MAX_FOUR_DIGIT_YEAR_MILLIS = parseToMillis("3000-01-01 00:00:00");
+
+ private static long parseToMillis(String s) {
+ try {
+ return DATE_FORMAT.get().parse(s).getTime();
+ } catch (ParseException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ public static Timestamp getRandTimestamp(Random r) {
+ return getRandTimestamp(r, MIN_YEAR, MAX_YEAR);
+ }
+
+ public static Timestamp getRandTimestamp(Random r, int minYear, int maxYear) {
+ String optionalNanos = "";
+ switch (r.nextInt(4)) {
+ case 0:
+ // No nanos.
+ break;
+ case 1:
+ optionalNanos = String.format(".%09d",
+ Integer.valueOf(r.nextInt((int) NANOSECONDS_PER_SECOND)));
+ break;
+ case 2:
+ // Limit to milliseconds only...
+ optionalNanos = String.format(".%09d",
+ Integer.valueOf(r.nextInt((int) MILLISECONDS_PER_SECOND)) * NANOSECONDS_PER_MILLISSECOND);
+ break;
+ case 3:
+ // Limit to below milliseconds only...
+ optionalNanos = String.format(".%09d",
+ Integer.valueOf(r.nextInt((int) NANOSECONDS_PER_MILLISSECOND)));
+ break;
+ }
+ String timestampStr = String.format("%04d-%02d-%02d %02d:%02d:%02d%s",
+ Integer.valueOf(minYear + r.nextInt(maxYear - minYear + 1)), // year
+ Integer.valueOf(1 + r.nextInt(12)), // month
+ Integer.valueOf(1 + r.nextInt(28)), // day
+ Integer.valueOf(0 + r.nextInt(24)), // hour
+ Integer.valueOf(0 + r.nextInt(60)), // minute
+ Integer.valueOf(0 + r.nextInt(60)), // second
+ optionalNanos);
+ Timestamp timestampVal;
+ try {
+ timestampVal = Timestamp.valueOf(timestampStr);
+ } catch (Exception e) {
+ System.err.println("Timestamp string " + timestampStr + " did not parse");
+ throw e;
+ }
+ return timestampVal;
+ }
+
+ public static long randomMillis(long minMillis, long maxMillis, Random rand) {
+ return minMillis + (long) ((maxMillis - minMillis) * rand.nextDouble());
+ }
+
+ public static long randomMillis(Random rand) {
+ return randomMillis(MIN_FOUR_DIGIT_YEAR_MILLIS, MAX_FOUR_DIGIT_YEAR_MILLIS, rand);
+ }
+
+ public static int randomNanos(Random rand, int decimalDigits) {
+ // Only keep the most significant decimalDigits digits.
+ int nanos = rand.nextInt((int) NANOSECONDS_PER_SECOND);
+ return nanos - nanos % (int) Math.pow(10, 9 - decimalDigits);
+ }
+
+ public static int randomNanos(Random rand) {
+ return randomNanos(rand, 9);
+ }
+}
\ No newline at end of file
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
new file mode 100644
index 0000000000000000000000000000000000000000..c4f19cf1df49bba789432aa40931ae27d6619f0e
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
@@ -0,0 +1,486 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+
+/**
+ * This class supports string and binary data by value reference -- i.e. each field is
+ * explicitly present, as opposed to provided by a dictionary reference.
+ * In some cases, all the values will be in the same byte array to begin with,
+ * but this need not be the case. If each value is in a separate byte
+ * array to start with, or not all of the values are in the same original
+ * byte array, you can still assign data by reference into this column vector.
+ * This gives flexibility to use this in multiple situations.
+ *
+ * When setting data by reference, the caller
+ * is responsible for allocating the byte arrays used to hold the data.
+ * You can also set data by value, as long as you call the initBuffer() method first.
+ * You can mix "by value" and "by reference" in the same column vector,
+ * though that use is probably not typical.
+ */
+public class BytesColumnVector extends ColumnVector {
+ public byte[][] vector;
+ public int[] start; // start offset of each field
+
+ /*
+ * The length of each field. If the value repeats for every entry, then it is stored
+ * in vector[0] and isRepeating from the superclass is set to true.
+ */
+ public int[] length;
+
+ // A call to increaseBufferSpace() or ensureValPreallocated() will ensure that buffer[] points to
+ // a byte[] with sufficient space for the specified size.
+ private byte[] buffer; // optional buffer to use when actually copying in data
+ private int nextFree; // next free position in buffer
+
+ // Hang onto a byte array for holding smaller byte values
+ private byte[] smallBuffer;
+ private int smallBufferNextFree;
+
+ private int bufferAllocationCount;
+
+ // Estimate that there will be 16 bytes per entry
+ static final int DEFAULT_BUFFER_SIZE = 16 * VectorizedRowBatch.DEFAULT_SIZE;
+
+ // Proportion of extra space to provide when allocating more buffer space.
+ static final float EXTRA_SPACE_FACTOR = (float) 1.2;
+
+ // Largest size allowed in smallBuffer
+ static final int MAX_SIZE_FOR_SMALL_BUFFER = 1024 * 1024;
+
+ /**
+ * Use this constructor for normal operation.
+ * All column vectors should be the default size normally.
+ */
+ public BytesColumnVector() {
+ this(VectorizedRowBatch.DEFAULT_SIZE);
+ }
+
+ /**
+ * Don't call this constructor except for testing purposes.
+ *
+ * @param size number of elements in the column vector
+ */
+ public BytesColumnVector(int size) {
+ super(Type.BYTES, size);
+ vector = new byte[size][];
+ start = new int[size];
+ length = new int[size];
+ }
+
+ /**
+ * Additional reset work for BytesColumnVector (releasing scratch bytes for by value strings).
+ */
+ @Override
+ public void reset() {
+ super.reset();
+ initBuffer(0);
+ }
+
+ /** Set a field by reference.
+ *
+ * @param elementNum index within column vector to set
+ * @param sourceBuf container of source data
+ * @param start start byte position within source
+ * @param length length of source byte sequence
+ */
+ public void setRef(int elementNum, byte[] sourceBuf, int start, int length) {
+ vector[elementNum] = sourceBuf;
+ this.start[elementNum] = start;
+ this.length[elementNum] = length;
+ }
+
+ /**
+ * You must call initBuffer first before using setVal().
+ * Provide the estimated number of bytes needed to hold
+ * a full column vector worth of byte string data.
+ *
+ * @param estimatedValueSize Estimated size of buffer space needed
+ */
+ public void initBuffer(int estimatedValueSize) {
+ nextFree = 0;
+ smallBufferNextFree = 0;
+
+ // if buffer is already allocated, keep using it, don't re-allocate
+ if (buffer != null) {
+ // Free up any previously allocated buffers that are referenced by vector
+ if (bufferAllocationCount > 0) {
+ for (int idx = 0; idx < vector.length; ++idx) {
+ vector[idx] = null;
+ }
+ buffer = smallBuffer; // In case last row was a large bytes value
+ }
+ } else {
+ // allocate a little extra space to limit need to re-allocate
+ int bufferSize = this.vector.length * (int)(estimatedValueSize * EXTRA_SPACE_FACTOR);
+ if (bufferSize < DEFAULT_BUFFER_SIZE) {
+ bufferSize = DEFAULT_BUFFER_SIZE;
+ }
+ buffer = new byte[bufferSize];
+ smallBuffer = buffer;
+ }
+ bufferAllocationCount = 0;
+ }
+
+ /**
+ * Initialize buffer to default size.
+ */
+ public void initBuffer() {
+ initBuffer(0);
+ }
+
+ /**
+ * @return amount of buffer space currently allocated
+ */
+ public int bufferSize() {
+ if (buffer == null) {
+ return 0;
+ }
+ return buffer.length;
+ }
+
+ /**
+ * Set a field by actually copying in to a local buffer.
+ * If you must actually copy data in to the array, use this method.
+ * DO NOT USE this method unless it's not practical to set data by reference with setRef().
+ * Setting data by reference tends to run a lot faster than copying data in.
+ *
+ * @param elementNum index within column vector to set
+ * @param sourceBuf container of source data
+ * @param start start byte position within source
+ * @param length length of source byte sequence
+ */
+ public void setVal(int elementNum, byte[] sourceBuf, int start, int length) {
+ if ((nextFree + length) > buffer.length) {
+ increaseBufferSpace(length);
+ }
+ System.arraycopy(sourceBuf, start, buffer, nextFree, length);
+ vector[elementNum] = buffer;
+ this.start[elementNum] = nextFree;
+ this.length[elementNum] = length;
+ nextFree += length;
+ }
+
+ /**
+ * Set a field by actually copying in to a local buffer.
+ * If you must actually copy data in to the array, use this method.
+ * DO NOT USE this method unless it's not practical to set data by reference with setRef().
+ * Setting data by reference tends to run a lot faster than copying data in.
+ *
+ * @param elementNum index within column vector to set
+ * @param sourceBuf container of source data
+ */
+ public void setVal(int elementNum, byte[] sourceBuf) {
+ setVal(elementNum, sourceBuf, 0, sourceBuf.length);
+ }
+
+ /**
+ * Preallocate space in the local buffer so the caller can fill in the value bytes themselves.
+ *
+ * Always use with getValPreallocatedBytes, getValPreallocatedStart, and setValPreallocated.
+ */
+ public void ensureValPreallocated(int length) {
+ if ((nextFree + length) > buffer.length) {
+ increaseBufferSpace(length);
+ }
+ }
+
+ public byte[] getValPreallocatedBytes() {
+ return buffer;
+ }
+
+ public int getValPreallocatedStart() {
+ return nextFree;
+ }
+
+ /**
+ * Set the length of the preallocated values bytes used.
+ * @param elementNum
+ * @param length
+ */
+ public void setValPreallocated(int elementNum, int length) {
+ vector[elementNum] = buffer;
+ this.start[elementNum] = nextFree;
+ this.length[elementNum] = length;
+ nextFree += length;
+ }
+
+ /**
+ * Set a field to the concatenation of two string values. Result data is copied
+ * into the internal buffer.
+ *
+ * @param elementNum index within column vector to set
+ * @param leftSourceBuf container of left argument
+ * @param leftStart start of left argument
+ * @param leftLen length of left argument
+ * @param rightSourceBuf container of right argument
+ * @param rightStart start of right argument
+ * @param rightLen length of right arugment
+ */
+ public void setConcat(int elementNum, byte[] leftSourceBuf, int leftStart, int leftLen,
+ byte[] rightSourceBuf, int rightStart, int rightLen) {
+ int newLen = leftLen + rightLen;
+ if ((nextFree + newLen) > buffer.length) {
+ increaseBufferSpace(newLen);
+ }
+ vector[elementNum] = buffer;
+ this.start[elementNum] = nextFree;
+ this.length[elementNum] = newLen;
+
+ System.arraycopy(leftSourceBuf, leftStart, buffer, nextFree, leftLen);
+ nextFree += leftLen;
+ System.arraycopy(rightSourceBuf, rightStart, buffer, nextFree, rightLen);
+ nextFree += rightLen;
+ }
+
+ /**
+ * Increase buffer space enough to accommodate next element.
+ * This uses an exponential increase mechanism to rapidly
+ * increase buffer size to enough to hold all data.
+ * As batches get re-loaded, buffer space allocated will quickly
+ * stabilize.
+ *
+ * @param nextElemLength size of next element to be added
+ */
+ public void increaseBufferSpace(int nextElemLength) {
+ // A call to increaseBufferSpace() or ensureValPreallocated() will ensure that buffer[] points to
+ // a byte[] with sufficient space for the specified size.
+ // This will either point to smallBuffer, or to a newly allocated byte array for larger values.
+
+ if (nextElemLength > MAX_SIZE_FOR_SMALL_BUFFER) {
+ // Larger allocations will be special-cased and will not use the normal buffer.
+ // buffer/nextFree will be set to a newly allocated array just for the current row.
+ // The next row will require another call to increaseBufferSpace() since this new buffer should be used up.
+ byte[] newBuffer = new byte[nextElemLength];
+ ++bufferAllocationCount;
+ // If the buffer was pointing to smallBuffer, then nextFree keeps track of the current state
+ // of the free index for smallBuffer. We now need to save this value to smallBufferNextFree
+ // so we don't lose this. A bit of a weird dance here.
+ if (smallBuffer == buffer) {
+ smallBufferNextFree = nextFree;
+ }
+ buffer = newBuffer;
+ nextFree = 0;
+ } else {
+ // This value should go into smallBuffer.
+ if (smallBuffer != buffer) {
+ // Previous row was for a large bytes value ( > MAX_SIZE_FOR_SMALL_BUFFER).
+ // Use smallBuffer if possible.
+ buffer = smallBuffer;
+ nextFree = smallBufferNextFree;
+ }
+
+ // smallBuffer might still be out of space
+ if ((nextFree + nextElemLength) > buffer.length) {
+ int newLength = smallBuffer.length * 2;
+ while (newLength < nextElemLength) {
+ if (newLength < 0) {
+ throw new RuntimeException("Overflow of newLength. smallBuffer.length="
+ + smallBuffer.length + ", nextElemLength=" + nextElemLength);
+ }
+ newLength *= 2;
+ }
+ smallBuffer = new byte[newLength];
+ ++bufferAllocationCount;
+ smallBufferNextFree = 0;
+ // Update buffer
+ buffer = smallBuffer;
+ nextFree = 0;
+ }
+ }
+ }
+
+ /** Copy the current object contents into the output. Only copy selected entries,
+ * as indicated by selectedInUse and the sel array.
+ */
+ public void copySelected(
+ boolean selectedInUse, int[] sel, int size, BytesColumnVector output) {
+
+ // Output has nulls if and only if input has nulls.
+ output.noNulls = noNulls;
+ output.isRepeating = false;
+
+ // Handle repeating case
+ if (isRepeating) {
+ output.setVal(0, vector[0], start[0], length[0]);
+ output.isNull[0] = isNull[0];
+ output.isRepeating = true;
+ return;
+ }
+
+ // Handle normal case
+
+ // Copy data values over
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.setVal(i, vector[i], start[i], length[i]);
+ }
+ }
+ else {
+ for (int i = 0; i < size; i++) {
+ output.setVal(i, vector[i], start[i], length[i]);
+ }
+ }
+
+ // Copy nulls over if needed
+ if (!noNulls) {
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.isNull[i] = isNull[i];
+ }
+ }
+ else {
+ System.arraycopy(isNull, 0, output.isNull, 0, size);
+ }
+ }
+ }
+
+ /** Simplify vector by brute-force flattening noNulls and isRepeating
+ * This can be used to reduce combinatorial explosion of code paths in VectorExpressions
+ * with many arguments, at the expense of loss of some performance.
+ */
+ public void flatten(boolean selectedInUse, int[] sel, int size) {
+ flattenPush();
+ if (isRepeating) {
+ isRepeating = false;
+
+ // setRef is used below and this is safe, because the reference
+ // is to data owned by this column vector. If this column vector
+ // gets re-used, the whole thing is re-used together so there
+ // is no danger of a dangling reference.
+
+ // Only copy data values if entry is not null. The string value
+ // at position 0 is undefined if the position 0 value is null.
+ if (noNulls || !isNull[0]) {
+
+ // loops start at position 1 because position 0 is already set
+ if (selectedInUse) {
+ for (int j = 1; j < size; j++) {
+ int i = sel[j];
+ this.setRef(i, vector[0], start[0], length[0]);
+ }
+ } else {
+ for (int i = 1; i < size; i++) {
+ this.setRef(i, vector[0], start[0], length[0]);
+ }
+ }
+ }
+ flattenRepeatingNulls(selectedInUse, sel, size);
+ }
+ flattenNoNulls(selectedInUse, sel, size);
+ }
+
+ // Fill the all the vector entries with provided value
+ public void fill(byte[] value) {
+ noNulls = true;
+ isRepeating = true;
+ setRef(0, value, 0, value.length);
+ }
+
+ // Fill the column vector with nulls
+ public void fillWithNulls() {
+ noNulls = false;
+ isRepeating = true;
+ vector[0] = null;
+ isNull[0] = true;
+ }
+
+ @Override
+ public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) {
+ if (inputVector.isRepeating) {
+ inputElementNum = 0;
+ }
+ if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) {
+ isNull[outElementNum] = false;
+ BytesColumnVector in = (BytesColumnVector) inputVector;
+ setVal(outElementNum, in.vector[inputElementNum],
+ in.start[inputElementNum], in.length[inputElementNum]);
+ } else {
+ isNull[outElementNum] = true;
+ noNulls = false;
+ }
+ }
+
+ @Override
+ public void init() {
+ initBuffer(0);
+ }
+
+ public String toString(int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ return new String(vector[row], start[row], length[row]);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ buffer.append('"');
+ buffer.append(new String(vector[row], start[row], length[row]));
+ buffer.append('"');
+ } else {
+ buffer.append("null");
+ }
+ }
+
+ @Override
+ public void ensureSize(int size, boolean preserveData) {
+ super.ensureSize(size, preserveData);
+ if (size > vector.length) {
+ int[] oldStart = start;
+ start = new int[size];
+ int[] oldLength = length;
+ length = new int[size];
+ byte[][] oldVector = vector;
+ vector = new byte[size][];
+ if (preserveData) {
+ if (isRepeating) {
+ vector[0] = oldVector[0];
+ start[0] = oldStart[0];
+ length[0] = oldLength[0];
+ } else {
+ System.arraycopy(oldVector, 0, vector, 0, oldVector.length);
+ System.arraycopy(oldStart, 0, start, 0 , oldStart.length);
+ System.arraycopy(oldLength, 0, length, 0, oldLength.length);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void shallowCopyTo(ColumnVector otherCv) {
+ BytesColumnVector other = (BytesColumnVector)otherCv;
+ super.shallowCopyTo(other);
+ other.nextFree = nextFree;
+ other.vector = vector;
+ other.start = start;
+ other.length = length;
+ other.buffer = buffer;
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
new file mode 100644
index 0000000000000000000000000000000000000000..57342729fe2b1d2e20d3e51088d59ac3fe4fbd83
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import java.util.Arrays;
+
+/**
+ * ColumnVector contains the shared structure for the sub-types,
+ * including NULL information, and whether this vector
+ * repeats, i.e. has all values the same, so only the first
+ * one is set. This is used to accelerate query performance
+ * by handling a whole vector in O(1) time when applicable.
+ *
+ * The fields are public by design since this is a performance-critical
+ * structure that is used in the inner loop of query execution.
+ */
+public abstract class ColumnVector {
+
+ /*
+ * The current kinds of column vectors.
+ */
+ public static enum Type {
+ NONE, // Useful when the type of column vector has not be determined yet.
+ LONG,
+ DOUBLE,
+ BYTES,
+ DECIMAL,
+ DECIMAL_64,
+ TIMESTAMP,
+ INTERVAL_DAY_TIME,
+ STRUCT,
+ LIST,
+ MAP,
+ UNION
+ }
+
+ public final Type type;
+
+ /*
+ * If hasNulls is true, then this array contains true if the value
+ * is null, otherwise false. The array is always allocated, so a batch can be re-used
+ * later and nulls added.
+ */
+ public boolean[] isNull;
+
+ // If the whole column vector has no nulls, this is true, otherwise false.
+ public boolean noNulls;
+
+ /*
+ * True if same value repeats for whole column vector.
+ * If so, vector[0] holds the repeating value.
+ */
+ public boolean isRepeating;
+
+ // Variables to hold state from before flattening so it can be easily restored.
+ private boolean preFlattenIsRepeating;
+ private boolean preFlattenNoNulls;
+
+ /**
+ * Constructor for super-class ColumnVector. This is not called directly,
+ * but used to initialize inherited fields.
+ *
+ * @param len Vector length
+ */
+ public ColumnVector(Type type, int len) {
+ this.type = type;
+ isNull = new boolean[len];
+ noNulls = true;
+ isRepeating = false;
+ preFlattenNoNulls = true;
+ preFlattenIsRepeating = false;
+ }
+
+ /**
+ * Resets the column to default state
+ * - fills the isNull array with false
+ * - sets noNulls to true
+ * - sets isRepeating to false
+ */
+ public void reset() {
+ if (!noNulls) {
+ Arrays.fill(isNull, false);
+ }
+ noNulls = true;
+ isRepeating = false;
+ preFlattenNoNulls = true;
+ preFlattenIsRepeating = false;
+ }
+
+ /**
+ * Sets the isRepeating flag. Recurses over structs and unions so that the
+ * flags are set correctly.
+ * @param isRepeating
+ */
+ public void setRepeating(boolean isRepeating) {
+ this.isRepeating = isRepeating;
+ }
+
+ abstract public void flatten(boolean selectedInUse, int[] sel, int size);
+
+ // Simplify vector by brute-force flattening noNulls if isRepeating
+ // This can be used to reduce combinatorial explosion of code paths in VectorExpressions
+ // with many arguments.
+ protected void flattenRepeatingNulls(boolean selectedInUse, int[] sel,
+ int size) {
+
+ boolean nullFillValue;
+
+ if (noNulls) {
+ nullFillValue = false;
+ } else {
+ nullFillValue = isNull[0];
+ }
+
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ isNull[i] = nullFillValue;
+ }
+ } else {
+ Arrays.fill(isNull, 0, size, nullFillValue);
+ }
+
+ // all nulls are now explicit
+ noNulls = false;
+ }
+
+ protected void flattenNoNulls(boolean selectedInUse, int[] sel,
+ int size) {
+ if (noNulls) {
+ noNulls = false;
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ isNull[sel[j]] = false;
+ }
+ } else {
+ Arrays.fill(isNull, 0, size, false);
+ }
+ }
+ }
+
+ /**
+ * Restore the state of isRepeating and noNulls to what it was
+ * before flattening. This must only be called just after flattening
+ * and then evaluating a VectorExpression on the column vector.
+ * It is an optimization that allows other operations on the same
+ * column to continue to benefit from the isRepeating and noNulls
+ * indicators.
+ */
+ public void unFlatten() {
+ isRepeating = preFlattenIsRepeating;
+ noNulls = preFlattenNoNulls;
+ }
+
+ // Record repeating and no nulls state to be restored later.
+ protected void flattenPush() {
+ preFlattenIsRepeating = isRepeating;
+ preFlattenNoNulls = noNulls;
+ }
+
+ /**
+ * Set the element in this column vector from the given input vector.
+ * This method can assume that the output does not have isRepeating set.
+ */
+ public abstract void setElement(int outElementNum, int inputElementNum,
+ ColumnVector inputVector);
+
+ /**
+ * Initialize the column vector. This method can be overridden by specific column vector types.
+ * Use this method only if the individual type of the column vector is not known, otherwise its
+ * preferable to call specific initialization methods.
+ */
+ public void init() {
+ // Do nothing by default
+ }
+
+ /**
+ * Ensure the ColumnVector can hold at least size values.
+ * This method is deliberately *not* recursive because the complex types
+ * can easily have more (or less) children than the upper levels.
+ * @param size the new minimum size
+ * @param preserveData should the old data be preserved?
+ */
+ public void ensureSize(int size, boolean preserveData) {
+ if (isNull.length < size) {
+ boolean[] oldArray = isNull;
+ isNull = new boolean[size];
+ if (preserveData && !noNulls) {
+ if (isRepeating) {
+ isNull[0] = oldArray[0];
+ } else {
+ System.arraycopy(oldArray, 0, isNull, 0, oldArray.length);
+ }
+ }
+ }
+ }
+
+ /**
+ * Print the value for this column into the given string builder.
+ * @param buffer the buffer to print into
+ * @param row the id of the row to print
+ */
+ public abstract void stringifyValue(StringBuilder buffer,
+ int row);
+
+ /**
+ * Shallow copy of the contents of this vector to the other vector;
+ * replaces other vector's values.
+ */
+ public void shallowCopyTo(ColumnVector otherCv) {
+ otherCv.isNull = isNull;
+ otherCv.noNulls = noNulls;
+ otherCv.isRepeating = isRepeating;
+ otherCv.preFlattenIsRepeating = preFlattenIsRepeating;
+ otherCv.preFlattenNoNulls = preFlattenNoNulls;
+ }
+ }
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java
new file mode 100644
index 0000000000000000000000000000000000000000..5548b9dfbfd08efac0119427165f2981fb4f2229
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+
+/**
+
+ */
+public class Decimal64ColumnVector extends LongColumnVector {
+
+ public short scale;
+ public short precision;
+
+ private HiveDecimalWritable tempHiveDecWritable;
+
+ public Decimal64ColumnVector(int precision, int scale) {
+ this(VectorizedRowBatch.DEFAULT_SIZE, precision, scale);
+ }
+
+ public Decimal64ColumnVector(int size, int precision, int scale) {
+ super(size);
+ this.precision = (short) precision;
+ this.scale = (short) scale;
+ tempHiveDecWritable = new HiveDecimalWritable();
+ }
+
+ public void set(int elementNum, HiveDecimalWritable writable) {
+ tempHiveDecWritable.set(writable);
+ tempHiveDecWritable.mutateEnforcePrecisionScale(precision, scale);
+ if (!tempHiveDecWritable.isSet()) {
+ noNulls = false;
+ isNull[elementNum] = true;
+ } else {
+ isNull[elementNum] = false;
+ vector[elementNum] = tempHiveDecWritable.serialize64(scale);
+ }
+ }
+
+ public void set(int elementNum, HiveDecimal hiveDec) {
+ tempHiveDecWritable.set(hiveDec);
+ tempHiveDecWritable.mutateEnforcePrecisionScale(precision, scale);
+ if (!tempHiveDecWritable.isSet()) {
+ noNulls = false;
+ isNull[elementNum] = true;
+ } else {
+ isNull[elementNum] = false;
+ vector[elementNum] = tempHiveDecWritable.serialize64(scale);
+ }
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
new file mode 100644
index 0000000000000000000000000000000000000000..cfe40aca4908701b8d71955f0d52b3cb1f1a72cd
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+
+public class DecimalColumnVector extends ColumnVector {
+
+ /**
+ * A vector of HiveDecimalWritable objects.
+ *
+ * For high performance and easy access to this low-level structure,
+ * the fields are public by design (as they are in other ColumnVector
+ * types).
+ */
+ public HiveDecimalWritable[] vector;
+ public short scale;
+ public short precision;
+
+ public DecimalColumnVector(int precision, int scale) {
+ this(VectorizedRowBatch.DEFAULT_SIZE, precision, scale);
+ }
+
+ public DecimalColumnVector(int size, int precision, int scale) {
+ super(Type.DECIMAL, size);
+ this.precision = (short) precision;
+ this.scale = (short) scale;
+ vector = new HiveDecimalWritable[size];
+ for (int i = 0; i < size; i++) {
+ vector[i] = new HiveDecimalWritable(0); // Initially zero.
+ }
+ }
+
+ // Fill the all the vector entries with provided value
+ public void fill(HiveDecimal value) {
+ noNulls = true;
+ isRepeating = true;
+ if (vector[0] == null) {
+ vector[0] = new HiveDecimalWritable(value);
+ } else {
+ vector[0].set(value);
+ }
+ }
+
+ @Override
+ public void flatten(boolean selectedInUse, int[] sel, int size) {
+ // TODO Auto-generated method stub
+ }
+
+ @Override
+ public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) {
+ if (inputVector.isRepeating) {
+ inputElementNum = 0;
+ }
+ if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) {
+ vector[outElementNum].set(
+ ((DecimalColumnVector) inputVector).vector[inputElementNum],
+ precision, scale);
+ if (!vector[outElementNum].isSet()) {
+ isNull[outElementNum] = true;
+ noNulls = false;
+ } else {
+ isNull[outElementNum] = false;
+ }
+ } else {
+ isNull[outElementNum] = true;
+ noNulls = false;
+ }
+ }
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ buffer.append(vector[row].toString());
+ } else {
+ buffer.append("null");
+ }
+ }
+
+ public void set(int elementNum, HiveDecimalWritable writeable) {
+ vector[elementNum].set(writeable, precision, scale);
+ if (!vector[elementNum].isSet()) {
+ noNulls = false;
+ isNull[elementNum] = true;
+ } else {
+ isNull[elementNum] = false;
+ }
+ }
+
+ public void set(int elementNum, HiveDecimal hiveDec) {
+ vector[elementNum].set(hiveDec, precision, scale);
+ if (!vector[elementNum].isSet()) {
+ noNulls = false;
+ isNull[elementNum] = true;
+ } else {
+ isNull[elementNum] = false;
+ }
+ }
+
+ public void setNullDataValue(int elementNum) {
+ // E.g. For scale 2 the minimum is "0.01"
+ vector[elementNum].setFromLongAndScale(1L, scale);
+ }
+
+ @Override
+ public void ensureSize(int size, boolean preserveData) {
+ super.ensureSize(size, preserveData);
+ if (size <= vector.length) return; // We assume the existing vector is always valid.
+ HiveDecimalWritable[] oldArray = vector;
+ vector = new HiveDecimalWritable[size];
+ int initPos = 0;
+ if (preserveData) {
+ // we copy all of the values to avoid creating more objects
+ // TODO: it might be cheaper to always preserve data or reset existing objects
+ initPos = oldArray.length;
+ System.arraycopy(oldArray, 0, vector, 0 , oldArray.length);
+ }
+ for (int i = initPos; i < vector.length; ++i) {
+ vector[i] = new HiveDecimalWritable(0); // Initially zero.
+ }
+ }
+
+ @Override
+ public void shallowCopyTo(ColumnVector otherCv) {
+ DecimalColumnVector other = (DecimalColumnVector)otherCv;
+ super.shallowCopyTo(other);
+ other.scale = scale;
+ other.precision = precision;
+ other.vector = vector;
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java
new file mode 100644
index 0000000000000000000000000000000000000000..1395144ddde598508995d0a1077b8c19d2d6c510
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java
@@ -0,0 +1,184 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a nullable double precision floating point column vector.
+ * This class will be used for operations on all floating point types (float, double)
+ * and as such will use a 64-bit double value to hold the biggest possible value.
+ * During copy-in/copy-out, smaller types (i.e. float) will be converted as needed. This will
+ * reduce the amount of code that needs to be generated and also will run fast since the
+ * machine operates with 64-bit words.
+ *
+ * The vector[] field is public by design for high-performance access in the inner
+ * loop of query execution.
+ */
+public class DoubleColumnVector extends ColumnVector {
+ public double[] vector;
+ public static final double NULL_VALUE = Double.NaN;
+
+ /**
+ * Use this constructor by default. All column vectors
+ * should normally be the default size.
+ */
+ public DoubleColumnVector() {
+ this(VectorizedRowBatch.DEFAULT_SIZE);
+ }
+
+ /**
+ * Don't use this except for testing purposes.
+ *
+ * @param len
+ */
+ public DoubleColumnVector(int len) {
+ super(Type.DOUBLE, len);
+ vector = new double[len];
+ }
+
+ // Copy the current object contents into the output. Only copy selected entries,
+ // as indicated by selectedInUse and the sel array.
+ public void copySelected(
+ boolean selectedInUse, int[] sel, int size, DoubleColumnVector output) {
+
+ // Output has nulls if and only if input has nulls.
+ output.noNulls = noNulls;
+ output.isRepeating = false;
+
+ // Handle repeating case
+ if (isRepeating) {
+ output.vector[0] = vector[0];
+ output.isNull[0] = isNull[0];
+ output.isRepeating = true;
+ return;
+ }
+
+ // Handle normal case
+
+ // Copy data values over
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.vector[i] = vector[i];
+ }
+ }
+ else {
+ System.arraycopy(vector, 0, output.vector, 0, size);
+ }
+
+ // Copy nulls over if needed
+ if (!noNulls) {
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.isNull[i] = isNull[i];
+ }
+ }
+ else {
+ System.arraycopy(isNull, 0, output.isNull, 0, size);
+ }
+ }
+ }
+
+ // Fill the column vector with the provided value
+ public void fill(double value) {
+ noNulls = true;
+ isRepeating = true;
+ vector[0] = value;
+ }
+
+ // Fill the column vector with nulls
+ public void fillWithNulls() {
+ noNulls = false;
+ isRepeating = true;
+ vector[0] = NULL_VALUE;
+ isNull[0] = true;
+ }
+
+ // Simplify vector by brute-force flattening noNulls and isRepeating
+ // This can be used to reduce combinatorial explosion of code paths in VectorExpressions
+ // with many arguments.
+ public void flatten(boolean selectedInUse, int[] sel, int size) {
+ flattenPush();
+ if (isRepeating) {
+ isRepeating = false;
+ double repeatVal = vector[0];
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ vector[i] = repeatVal;
+ }
+ } else {
+ Arrays.fill(vector, 0, size, repeatVal);
+ }
+ flattenRepeatingNulls(selectedInUse, sel, size);
+ }
+ flattenNoNulls(selectedInUse, sel, size);
+ }
+
+ @Override
+ public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) {
+ if (inputVector.isRepeating) {
+ inputElementNum = 0;
+ }
+ if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) {
+ isNull[outElementNum] = false;
+ vector[outElementNum] =
+ ((DoubleColumnVector) inputVector).vector[inputElementNum];
+ } else {
+ isNull[outElementNum] = true;
+ noNulls = false;
+ }
+ }
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ buffer.append(vector[row]);
+ } else {
+ buffer.append("null");
+ }
+ }
+
+ @Override
+ public void ensureSize(int size, boolean preserveData) {
+ super.ensureSize(size, preserveData);
+ if (size > vector.length) {
+ double[] oldArray = vector;
+ vector = new double[size];
+ if (preserveData) {
+ if (isRepeating) {
+ vector[0] = oldArray[0];
+ } else {
+ System.arraycopy(oldArray, 0, vector, 0 , oldArray.length);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void shallowCopyTo(ColumnVector otherCv) {
+ DoubleColumnVector other = (DoubleColumnVector)otherCv;
+ super.shallowCopyTo(other);
+ other.vector = vector;
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java
new file mode 100644
index 0000000000000000000000000000000000000000..587e2b9ac8454174109c3b437e539876bc5b6ce0
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java
@@ -0,0 +1,375 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import java.util.Arrays;
+
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This class represents a nullable interval day time column vector capable of handing a
+ * wide range of interval day time values.
+ *
+ * We store the 2 (value) fields of a HiveIntervalDayTime class in primitive arrays.
+ *
+ * We do this to avoid an array of Java HiveIntervalDayTime objects which would have poor storage
+ * and memory access characteristics.
+ *
+ * Generally, the caller will fill in a scratch HiveIntervalDayTime object with values from a row,
+ * work using the scratch HiveIntervalDayTime, and then perhaps update the column vector row
+ * with a result.
+ */
+public class IntervalDayTimeColumnVector extends ColumnVector {
+
+ /*
+ * The storage arrays for this column vector corresponds to the storage of a HiveIntervalDayTime:
+ */
+ private long[] totalSeconds;
+ // The values from HiveIntervalDayTime.getTotalSeconds().
+
+ private int[] nanos;
+ // The values from HiveIntervalDayTime.getNanos().
+
+ /*
+ * Scratch objects.
+ */
+ private final HiveIntervalDayTime scratchIntervalDayTime;
+
+ private Writable scratchWritable;
+ // Supports keeping a HiveIntervalDayTimeWritable object without having to import
+ // that definition...
+
+ /**
+ * Use this constructor by default. All column vectors
+ * should normally be the default size.
+ */
+ public IntervalDayTimeColumnVector() {
+ this(VectorizedRowBatch.DEFAULT_SIZE);
+ }
+
+ /**
+ * Don't use this except for testing purposes.
+ *
+ * @param len the number of rows
+ */
+ public IntervalDayTimeColumnVector(int len) {
+ super(Type.INTERVAL_DAY_TIME, len);
+
+ totalSeconds = new long[len];
+ nanos = new int[len];
+
+ scratchIntervalDayTime = new HiveIntervalDayTime();
+
+ scratchWritable = null; // Allocated by caller.
+ }
+
+ /**
+ * Return the number of rows.
+ * @return
+ */
+ public int getLength() {
+ return totalSeconds.length;
+ }
+
+ /**
+ * Return a row's HiveIntervalDayTime.getTotalSeconds() value.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @return
+ */
+ public long getTotalSeconds(int elementNum) {
+ return totalSeconds[elementNum];
+ }
+
+ /**
+ * Return a row's HiveIntervalDayTime.getNanos() value.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @return
+ */
+ public long getNanos(int elementNum) {
+ return nanos[elementNum];
+ }
+
+ /**
+ * Return a row's HiveIntervalDayTime.getDouble() value.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @return
+ */
+ public double getDouble(int elementNum) {
+ return asScratchIntervalDayTime(elementNum).getDouble();
+ }
+
+ /**
+ * Set a HiveIntervalDayTime object from a row of the column.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param intervalDayTime
+ * @param elementNum
+ */
+ public void intervalDayTimeUpdate(HiveIntervalDayTime intervalDayTime, int elementNum) {
+ intervalDayTime.set(totalSeconds[elementNum], nanos[elementNum]);
+ }
+
+
+ /**
+ * Return the scratch HiveIntervalDayTime object set from a row.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @return
+ */
+ public HiveIntervalDayTime asScratchIntervalDayTime(int elementNum) {
+ scratchIntervalDayTime.set(totalSeconds[elementNum], nanos[elementNum]);
+ return scratchIntervalDayTime;
+ }
+
+ /**
+ * Return the scratch HiveIntervalDayTime (contents undefined).
+ * @return
+ */
+ public HiveIntervalDayTime getScratchIntervalDayTime() {
+ return scratchIntervalDayTime;
+ }
+
+ /**
+ * Compare row to HiveIntervalDayTime.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @param intervalDayTime
+ * @return -1, 0, 1 standard compareTo values.
+ */
+ public int compareTo(int elementNum, HiveIntervalDayTime intervalDayTime) {
+ return asScratchIntervalDayTime(elementNum).compareTo(intervalDayTime);
+ }
+
+ /**
+ * Compare HiveIntervalDayTime to row.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param intervalDayTime
+ * @param elementNum
+ * @return -1, 0, 1 standard compareTo values.
+ */
+ public int compareTo(HiveIntervalDayTime intervalDayTime, int elementNum) {
+ return intervalDayTime.compareTo(asScratchIntervalDayTime(elementNum));
+ }
+
+ /**
+ * Compare a row to another TimestampColumnVector's row.
+ * @param elementNum1
+ * @param intervalDayTimeColVector2
+ * @param elementNum2
+ * @return
+ */
+ public int compareTo(int elementNum1, IntervalDayTimeColumnVector intervalDayTimeColVector2,
+ int elementNum2) {
+ return asScratchIntervalDayTime(elementNum1).compareTo(
+ intervalDayTimeColVector2.asScratchIntervalDayTime(elementNum2));
+ }
+
+ /**
+ * Compare another TimestampColumnVector's row to a row.
+ * @param intervalDayTimeColVector1
+ * @param elementNum1
+ * @param elementNum2
+ * @return
+ */
+ public int compareTo(IntervalDayTimeColumnVector intervalDayTimeColVector1, int elementNum1,
+ int elementNum2) {
+ return intervalDayTimeColVector1.asScratchIntervalDayTime(elementNum1).compareTo(
+ asScratchIntervalDayTime(elementNum2));
+ }
+
+ @Override
+ public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) {
+
+ IntervalDayTimeColumnVector timestampColVector = (IntervalDayTimeColumnVector) inputVector;
+
+ totalSeconds[outElementNum] = timestampColVector.totalSeconds[inputElementNum];
+ nanos[outElementNum] = timestampColVector.nanos[inputElementNum];
+ }
+
+ // Simplify vector by brute-force flattening noNulls and isRepeating
+ // This can be used to reduce combinatorial explosion of code paths in VectorExpressions
+ // with many arguments.
+ public void flatten(boolean selectedInUse, int[] sel, int size) {
+ flattenPush();
+ if (isRepeating) {
+ isRepeating = false;
+ long repeatFastTime = totalSeconds[0];
+ int repeatNanos = nanos[0];
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ totalSeconds[i] = repeatFastTime;
+ nanos[i] = repeatNanos;
+ }
+ } else {
+ Arrays.fill(totalSeconds, 0, size, repeatFastTime);
+ Arrays.fill(nanos, 0, size, repeatNanos);
+ }
+ flattenRepeatingNulls(selectedInUse, sel, size);
+ }
+ flattenNoNulls(selectedInUse, sel, size);
+ }
+
+ /**
+ * Set a row from a HiveIntervalDayTime.
+ * We assume the entry has already been isRepeated adjusted.
+ * @param elementNum
+ * @param intervalDayTime
+ */
+ public void set(int elementNum, HiveIntervalDayTime intervalDayTime) {
+ this.totalSeconds[elementNum] = intervalDayTime.getTotalSeconds();
+ this.nanos[elementNum] = intervalDayTime.getNanos();
+ }
+
+ /**
+ * Set a row from the current value in the scratch interval day time.
+ * @param elementNum
+ */
+ public void setFromScratchIntervalDayTime(int elementNum) {
+ this.totalSeconds[elementNum] = scratchIntervalDayTime.getTotalSeconds();
+ this.nanos[elementNum] = scratchIntervalDayTime.getNanos();
+ }
+
+ /**
+ * Set row to standard null value(s).
+ * We assume the entry has already been isRepeated adjusted.
+ * @param elementNum
+ */
+ public void setNullValue(int elementNum) {
+ totalSeconds[elementNum] = 0;
+ nanos[elementNum] = 1;
+ }
+
+ // Copy the current object contents into the output. Only copy selected entries,
+ // as indicated by selectedInUse and the sel array.
+ public void copySelected(
+ boolean selectedInUse, int[] sel, int size, IntervalDayTimeColumnVector output) {
+
+ // Output has nulls if and only if input has nulls.
+ output.noNulls = noNulls;
+ output.isRepeating = false;
+
+ // Handle repeating case
+ if (isRepeating) {
+ output.totalSeconds[0] = totalSeconds[0];
+ output.nanos[0] = nanos[0];
+ output.isNull[0] = isNull[0];
+ output.isRepeating = true;
+ return;
+ }
+
+ // Handle normal case
+
+ // Copy data values over
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.totalSeconds[i] = totalSeconds[i];
+ output.nanos[i] = nanos[i];
+ }
+ }
+ else {
+ System.arraycopy(totalSeconds, 0, output.totalSeconds, 0, size);
+ System.arraycopy(nanos, 0, output.nanos, 0, size);
+ }
+
+ // Copy nulls over if needed
+ if (!noNulls) {
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.isNull[i] = isNull[i];
+ }
+ }
+ else {
+ System.arraycopy(isNull, 0, output.isNull, 0, size);
+ }
+ }
+ }
+
+ /**
+ * Fill all the vector entries with a HiveIntervalDayTime.
+ * @param intervalDayTime
+ */
+ public void fill(HiveIntervalDayTime intervalDayTime) {
+ noNulls = true;
+ isRepeating = true;
+ totalSeconds[0] = intervalDayTime.getTotalSeconds();
+ nanos[0] = intervalDayTime.getNanos();
+ }
+
+ /**
+ * Return a convenience writable object stored by this column vector.
+ * Supports keeping a TimestampWritable object without having to import that definition...
+ * @return
+ */
+ public Writable getScratchWritable() {
+ return scratchWritable;
+ }
+
+ /**
+ * Set the convenience writable object stored by this column vector
+ * @param scratchWritable
+ */
+ public void setScratchWritable(Writable scratchWritable) {
+ this.scratchWritable = scratchWritable;
+ }
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ scratchIntervalDayTime.set(totalSeconds[row], nanos[row]);
+ buffer.append(scratchIntervalDayTime.toString());
+ } else {
+ buffer.append("null");
+ }
+ }
+
+ @Override
+ public void ensureSize(int size, boolean preserveData) {
+ super.ensureSize(size, preserveData);
+ if (size <= totalSeconds.length) return;
+ long[] oldTime = totalSeconds;
+ int[] oldNanos = nanos;
+ totalSeconds = new long[size];
+ nanos = new int[size];
+ if (preserveData) {
+ if (isRepeating) {
+ totalSeconds[0] = oldTime[0];
+ nanos[0] = oldNanos[0];
+ } else {
+ System.arraycopy(oldTime, 0, totalSeconds, 0, oldTime.length);
+ System.arraycopy(oldNanos, 0, nanos, 0, oldNanos.length);
+ }
+ }
+ }
+
+ @Override
+ public void shallowCopyTo(ColumnVector otherCv) {
+ IntervalDayTimeColumnVector other = (IntervalDayTimeColumnVector)otherCv;
+ super.shallowCopyTo(other);
+ other.totalSeconds = totalSeconds;
+ other.nanos = nanos;
+ }
+}
\ No newline at end of file
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ListColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ListColumnVector.java
new file mode 100644
index 0000000000000000000000000000000000000000..02a8b3cec5f5cad98e4802e6e1f1e9ece268921f
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ListColumnVector.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+/**
+ * The representation of a vectorized column of list objects.
+ *
+ * Each list is composed of a range of elements in the underlying child
+ * ColumnVector. The range for list i is
+ * offsets[i]..offsets[i]+lengths[i]-1 inclusive.
+ */
+public class ListColumnVector extends MultiValuedColumnVector {
+
+ public ColumnVector child;
+
+ public ListColumnVector() {
+ this(VectorizedRowBatch.DEFAULT_SIZE, null);
+ }
+
+ /**
+ * Constructor for ListColumnVector.
+ *
+ * @param len Vector length
+ * @param child The child vector
+ */
+ public ListColumnVector(int len, ColumnVector child) {
+ super(Type.LIST, len);
+ this.child = child;
+ }
+
+ @Override
+ protected void childFlatten(boolean useSelected, int[] selected, int size) {
+ child.flatten(useSelected, selected, size);
+ }
+
+ @Override
+ public void setElement(int outElementNum, int inputElementNum,
+ ColumnVector inputVector) {
+ ListColumnVector input = (ListColumnVector) inputVector;
+ if (input.isRepeating) {
+ inputElementNum = 0;
+ }
+ if (!input.noNulls && input.isNull[inputElementNum]) {
+ isNull[outElementNum] = true;
+ noNulls = false;
+ } else {
+ isNull[outElementNum] = false;
+ int offset = childCount;
+ int length = (int) input.lengths[inputElementNum];
+ int inputOffset = (int) input.offsets[inputElementNum];
+ offsets[outElementNum] = offset;
+ childCount += length;
+ lengths[outElementNum] = length;
+ child.ensureSize(childCount, true);
+ for (int i = 0; i < length; ++i) {
+ child.setElement(i + offset, inputOffset + i, input.child);
+ }
+ }
+ }
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ buffer.append('[');
+ boolean isFirst = true;
+ for(long i=offsets[row]; i < offsets[row] + lengths[row]; ++i) {
+ if (isFirst) {
+ isFirst = false;
+ } else {
+ buffer.append(", ");
+ }
+ child.stringifyValue(buffer, (int) i);
+ }
+ buffer.append(']');
+ } else {
+ buffer.append("null");
+ }
+ }
+
+ @Override
+ public void init() {
+ super.init();
+ child.init();
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ child.reset();
+ }
+
+ @Override
+ public void unFlatten() {
+ super.unFlatten();
+ if (!isRepeating || noNulls || !isNull[0]) {
+ child.unFlatten();
+ }
+ }
+
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java
new file mode 100644
index 0000000000000000000000000000000000000000..a407bcdaaaa84e957725783cf84fec06ddcf7143
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java
@@ -0,0 +1,230 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a nullable int column vector.
+ * This class will be used for operations on all integer types (tinyint, smallint, int, bigint)
+ * and as such will use a 64-bit long value to hold the biggest possible value.
+ * During copy-in/copy-out, smaller int types will be converted as needed. This will
+ * reduce the amount of code that needs to be generated and also will run fast since the
+ * machine operates with 64-bit words.
+ *
+ * The vector[] field is public by design for high-performance access in the inner
+ * loop of query execution.
+ */
+public class LongColumnVector extends ColumnVector {
+ public long[] vector;
+ public static final long NULL_VALUE = 1;
+
+ /**
+ * Use this constructor by default. All column vectors
+ * should normally be the default size.
+ */
+ public LongColumnVector() {
+ this(VectorizedRowBatch.DEFAULT_SIZE);
+ }
+
+ /**
+ * Don't use this except for testing purposes.
+ *
+ * @param len the number of rows
+ */
+ public LongColumnVector(int len) {
+ super(Type.LONG, len);
+ vector = new long[len];
+ }
+
+ // Copy the current object contents into the output. Only copy selected entries,
+ // as indicated by selectedInUse and the sel array.
+ public void copySelected(
+ boolean selectedInUse, int[] sel, int size, LongColumnVector output) {
+
+ // Output has nulls if and only if input has nulls.
+ output.noNulls = noNulls;
+ output.isRepeating = false;
+
+ // Handle repeating case
+ if (isRepeating) {
+ output.vector[0] = vector[0];
+ output.isNull[0] = isNull[0];
+ output.isRepeating = true;
+ return;
+ }
+
+ // Handle normal case
+
+ // Copy data values over
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.vector[i] = vector[i];
+ }
+ }
+ else {
+ System.arraycopy(vector, 0, output.vector, 0, size);
+ }
+
+ // Copy nulls over if needed
+ if (!noNulls) {
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.isNull[i] = isNull[i];
+ }
+ }
+ else {
+ System.arraycopy(isNull, 0, output.isNull, 0, size);
+ }
+ }
+ }
+
+ // Copy the current object contents into the output. Only copy selected entries,
+ // as indicated by selectedInUse and the sel array.
+ public void copySelected(
+ boolean selectedInUse, int[] sel, int size, DoubleColumnVector output) {
+
+ // Output has nulls if and only if input has nulls.
+ output.noNulls = noNulls;
+ output.isRepeating = false;
+
+ // Handle repeating case
+ if (isRepeating) {
+ output.vector[0] = vector[0]; // automatic conversion to double is done here
+ output.isNull[0] = isNull[0];
+ output.isRepeating = true;
+ return;
+ }
+
+ // Handle normal case
+
+ // Copy data values over
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.vector[i] = vector[i];
+ }
+ }
+ else {
+ for(int i = 0; i < size; ++i) {
+ output.vector[i] = vector[i];
+ }
+ }
+
+ // Copy nulls over if needed
+ if (!noNulls) {
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.isNull[i] = isNull[i];
+ }
+ }
+ else {
+ System.arraycopy(isNull, 0, output.isNull, 0, size);
+ }
+ }
+ }
+
+ // Fill the column vector with the provided value
+ public void fill(long value) {
+ noNulls = true;
+ isRepeating = true;
+ vector[0] = value;
+ }
+
+ // Fill the column vector with nulls
+ public void fillWithNulls() {
+ noNulls = false;
+ isRepeating = true;
+ vector[0] = NULL_VALUE;
+ isNull[0] = true;
+ }
+
+ // Simplify vector by brute-force flattening noNulls and isRepeating
+ // This can be used to reduce combinatorial explosion of code paths in VectorExpressions
+ // with many arguments.
+ public void flatten(boolean selectedInUse, int[] sel, int size) {
+ flattenPush();
+ if (isRepeating) {
+ isRepeating = false;
+ long repeatVal = vector[0];
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ vector[i] = repeatVal;
+ }
+ } else {
+ Arrays.fill(vector, 0, size, repeatVal);
+ }
+ flattenRepeatingNulls(selectedInUse, sel, size);
+ }
+ flattenNoNulls(selectedInUse, sel, size);
+ }
+
+ @Override
+ public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) {
+ if (inputVector.isRepeating) {
+ inputElementNum = 0;
+ }
+ if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) {
+ isNull[outElementNum] = false;
+ vector[outElementNum] =
+ ((LongColumnVector) inputVector).vector[inputElementNum];
+ } else {
+ isNull[outElementNum] = true;
+ noNulls = false;
+ }
+ }
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ buffer.append(vector[row]);
+ } else {
+ buffer.append("null");
+ }
+ }
+
+ @Override
+ public void ensureSize(int size, boolean preserveData) {
+ super.ensureSize(size, preserveData);
+ if (size > vector.length) {
+ long[] oldArray = vector;
+ vector = new long[size];
+ if (preserveData) {
+ if (isRepeating) {
+ vector[0] = oldArray[0];
+ } else {
+ System.arraycopy(oldArray, 0, vector, 0 , oldArray.length);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void shallowCopyTo(ColumnVector otherCv) {
+ LongColumnVector other = (LongColumnVector)otherCv;
+ super.shallowCopyTo(other);
+ other.vector = vector;
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MapColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MapColumnVector.java
new file mode 100644
index 0000000000000000000000000000000000000000..64badb99e5385af359691bc1d4df5119c9ffaac3
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MapColumnVector.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+/**
+ * The representation of a vectorized column of map objects.
+ *
+ * Each map is composed of a range of elements in the underlying child
+ * ColumnVector. The range for map i is
+ * offsets[i]..offsets[i]+lengths[i]-1 inclusive.
+ */
+public class MapColumnVector extends MultiValuedColumnVector {
+
+ public ColumnVector keys;
+ public ColumnVector values;
+
+ public MapColumnVector() {
+ this(VectorizedRowBatch.DEFAULT_SIZE, null, null);
+ }
+
+ /**
+ * Constructor for MapColumnVector
+ *
+ * @param len Vector length
+ * @param keys The keys column vector
+ * @param values The values column vector
+ */
+ public MapColumnVector(int len, ColumnVector keys, ColumnVector values) {
+ super(Type.MAP, len);
+ this.keys = keys;
+ this.values = values;
+ }
+
+ @Override
+ protected void childFlatten(boolean useSelected, int[] selected, int size) {
+ keys.flatten(useSelected, selected, size);
+ values.flatten(useSelected, selected, size);
+ }
+
+ @Override
+ public void setElement(int outElementNum, int inputElementNum,
+ ColumnVector inputVector) {
+ if (inputVector.isRepeating) {
+ inputElementNum = 0;
+ }
+ if (!inputVector.noNulls && inputVector.isNull[inputElementNum]) {
+ isNull[outElementNum] = true;
+ noNulls = false;
+ } else {
+ MapColumnVector input = (MapColumnVector) inputVector;
+ isNull[outElementNum] = false;
+ int offset = childCount;
+ int length = (int) input.lengths[inputElementNum];
+ int inputOffset = (int) input.offsets[inputElementNum];
+ offsets[outElementNum] = offset;
+ childCount += length;
+ lengths[outElementNum] = length;
+ keys.ensureSize(childCount, true);
+ values.ensureSize(childCount, true);
+ for (int i = 0; i < length; ++i) {
+ keys.setElement(i + offset, inputOffset + i, input.keys);
+ values.setElement(i + offset, inputOffset + i, input.values);
+ }
+ }
+ }
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ buffer.append('[');
+ boolean isFirst = true;
+ for(long i=offsets[row]; i < offsets[row] + lengths[row]; ++i) {
+ if (isFirst) {
+ isFirst = false;
+ } else {
+ buffer.append(", ");
+ }
+ buffer.append("{\"key\": ");
+ keys.stringifyValue(buffer, (int) i);
+ buffer.append(", \"value\": ");
+ values.stringifyValue(buffer, (int) i);
+ buffer.append('}');
+ }
+ buffer.append(']');
+ } else {
+ buffer.append("null");
+ }
+ }
+
+ @Override
+ public void init() {
+ super.init();
+ keys.init();
+ values.init();
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ keys.reset();
+ values.reset();
+ }
+
+ @Override
+ public void unFlatten() {
+ super.unFlatten();
+ if (!isRepeating || noNulls || !isNull[0]) {
+ keys.unFlatten();
+ values.unFlatten();
+ }
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MultiValuedColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MultiValuedColumnVector.java
new file mode 100644
index 0000000000000000000000000000000000000000..66136351d4f508d1dc7cf419da8eca41efa698a1
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/MultiValuedColumnVector.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import java.util.Arrays;
+
+/**
+ * The representation of a vectorized column of multi-valued objects, such
+ * as lists and maps.
+ *
+ * Each object is composed of a range of elements in the underlying child
+ * ColumnVector. The range for list i is
+ * offsets[i]..offsets[i]+lengths[i]-1 inclusive.
+ */
+public abstract class MultiValuedColumnVector extends ColumnVector {
+
+ public long[] offsets;
+ public long[] lengths;
+ // the number of children slots used
+ public int childCount;
+
+ /**
+ * Constructor for MultiValuedColumnVector.
+ *
+ * @param len Vector length
+ */
+ public MultiValuedColumnVector(Type type, int len) {
+ super(type, len);
+ childCount = 0;
+ offsets = new long[len];
+ lengths = new long[len];
+ }
+
+ protected abstract void childFlatten(boolean useSelected, int[] selected,
+ int size);
+
+ @Override
+ public void flatten(boolean selectedInUse, int[] sel, int size) {
+ flattenPush();
+
+ if (isRepeating) {
+ if (noNulls || !isNull[0]) {
+ if (selectedInUse) {
+ for (int i = 0; i < size; ++i) {
+ int row = sel[i];
+ offsets[row] = offsets[0];
+ lengths[row] = lengths[0];
+ isNull[row] = false;
+ }
+ } else {
+ Arrays.fill(offsets, 0, size, offsets[0]);
+ Arrays.fill(lengths, 0, size, lengths[0]);
+ Arrays.fill(isNull, 0, size, false);
+ }
+ // We optimize by assuming that a repeating list/map will run from
+ // from 0 .. lengths[0] in the child vector.
+ // Sanity check the assumption that we can start at 0.
+ if (offsets[0] != 0) {
+ throw new IllegalArgumentException("Repeating offset isn't 0, but " +
+ offsets[0]);
+ }
+ childFlatten(false, null, (int) lengths[0]);
+ } else {
+ if (selectedInUse) {
+ for(int i=0; i < size; ++i) {
+ isNull[sel[i]] = true;
+ }
+ } else {
+ Arrays.fill(isNull, 0, size, true);
+ }
+ }
+ isRepeating = false;
+ noNulls = false;
+ } else {
+ if (selectedInUse) {
+ int childSize = 0;
+ for(int i=0; i < size; ++i) {
+ childSize += lengths[sel[i]];
+ }
+ int[] childSelection = new int[childSize];
+ int idx = 0;
+ for(int i=0; i < size; ++i) {
+ int row = sel[i];
+ for(int elem=0; elem < lengths[row]; ++elem) {
+ childSelection[idx++] = (int) (offsets[row] + elem);
+ }
+ }
+ childFlatten(true, childSelection, childSize);
+ } else {
+ childFlatten(false, null, childCount);
+ }
+ flattenNoNulls(selectedInUse, sel, size);
+ }
+ }
+
+ @Override
+ public void ensureSize(int size, boolean preserveData) {
+ super.ensureSize(size, preserveData);
+ if (size > offsets.length) {
+ long[] oldOffsets = offsets;
+ offsets = new long[size];
+ long oldLengths[] = lengths;
+ lengths = new long[size];
+ if (preserveData) {
+ if (isRepeating) {
+ offsets[0] = oldOffsets[0];
+ lengths[0] = oldLengths[0];
+ } else {
+ System.arraycopy(oldOffsets, 0, offsets, 0 , oldOffsets.length);
+ System.arraycopy(oldLengths, 0, lengths, 0, oldLengths.length);
+ }
+ }
+ }
+ }
+
+ /**
+ * Initializee the vector
+ */
+ @Override
+ public void init() {
+ super.init();
+ childCount = 0;
+ }
+
+ /**
+ * Reset the vector for the next batch.
+ */
+ @Override
+ public void reset() {
+ super.reset();
+ childCount = 0;
+ }
+
+ @Override
+ public void shallowCopyTo(ColumnVector otherCv) {
+ throw new UnsupportedOperationException(); // Implement in future, if needed.
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java
new file mode 100644
index 0000000000000000000000000000000000000000..45f3ac6d6eb0969c96d461f0cf532f0a783ab61d
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+/**
+ * The representation of a vectorized column of struct objects.
+ *
+ * Each field is represented by a separate inner ColumnVector. Since this
+ * ColumnVector doesn't own any per row data other that the isNull flag, the
+ * isRepeating only covers the isNull array.
+ */
+public class StructColumnVector extends ColumnVector {
+
+ public ColumnVector[] fields;
+
+ public StructColumnVector() {
+ this(VectorizedRowBatch.DEFAULT_SIZE);
+ }
+
+ /**
+ * Constructor for StructColumnVector
+ *
+ * @param len Vector length
+ * @param fields the field column vectors
+ */
+ public StructColumnVector(int len, ColumnVector... fields) {
+ super(Type.STRUCT, len);
+ this.fields = fields;
+ }
+
+ @Override
+ public void flatten(boolean selectedInUse, int[] sel, int size) {
+ flattenPush();
+ for(int i=0; i < fields.length; ++i) {
+ fields[i].flatten(selectedInUse, sel, size);
+ }
+ flattenNoNulls(selectedInUse, sel, size);
+ }
+
+ @Override
+ public void setElement(int outElementNum, int inputElementNum,
+ ColumnVector inputVector) {
+ if (inputVector.isRepeating) {
+ inputElementNum = 0;
+ }
+ if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) {
+ isNull[outElementNum] = false;
+ ColumnVector[] inputFields = ((StructColumnVector) inputVector).fields;
+ for (int i = 0; i < inputFields.length; ++i) {
+ fields[i].setElement(outElementNum, inputElementNum, inputFields[i]);
+ }
+ } else {
+ noNulls = false;
+ isNull[outElementNum] = true;
+ }
+ }
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ buffer.append('[');
+ for(int i=0; i < fields.length; ++i) {
+ if (i != 0) {
+ buffer.append(", ");
+ }
+ fields[i].stringifyValue(buffer, row);
+ }
+ buffer.append(']');
+ } else {
+ buffer.append("null");
+ }
+ }
+
+ @Override
+ public void ensureSize(int size, boolean preserveData) {
+ super.ensureSize(size, preserveData);
+ for(int i=0; i < fields.length; ++i) {
+ fields[i].ensureSize(size, preserveData);
+ }
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ for(int i =0; i < fields.length; ++i) {
+ fields[i].reset();
+ }
+ }
+
+ @Override
+ public void init() {
+ super.init();
+ for(int i =0; i < fields.length; ++i) {
+ fields[i].init();
+ }
+ }
+
+ @Override
+ public void unFlatten() {
+ super.unFlatten();
+ for(int i=0; i < fields.length; ++i) {
+ fields[i].unFlatten();
+ }
+ }
+
+ @Override
+ public void setRepeating(boolean isRepeating) {
+ super.setRepeating(isRepeating);
+ for(int i=0; i < fields.length; ++i) {
+ fields[i].setRepeating(isRepeating);
+ }
+ }
+
+ @Override
+ public void shallowCopyTo(ColumnVector otherCv) {
+ throw new UnsupportedOperationException(); // Implement if needed.
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
new file mode 100644
index 0000000000000000000000000000000000000000..ef1c817efaac126aab0a35b013aa273c1a70f49f
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
@@ -0,0 +1,427 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import java.sql.Timestamp;
+import java.util.Arrays;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This class represents a nullable timestamp column vector capable of handing a wide range of
+ * timestamp values.
+ *
+ * We store the 2 (value) fields of a Timestamp class in primitive arrays.
+ *
+ * We do this to avoid an array of Java Timestamp objects which would have poor storage
+ * and memory access characteristics.
+ *
+ * Generally, the caller will fill in a scratch timestamp object with values from a row, work
+ * using the scratch timestamp, and then perhaps update the column vector row with a result.
+ */
+public class TimestampColumnVector extends ColumnVector {
+
+ /*
+ * The storage arrays for this column vector corresponds to the storage of a Timestamp:
+ */
+ public long[] time;
+ // The values from Timestamp.getTime().
+
+ public int[] nanos;
+ // The values from Timestamp.getNanos().
+
+ /*
+ * Scratch objects.
+ */
+ private final Timestamp scratchTimestamp;
+
+ private Writable scratchWritable;
+ // Supports keeping a TimestampWritable object without having to import that definition...
+
+ /**
+ * Use this constructor by default. All column vectors
+ * should normally be the default size.
+ */
+ public TimestampColumnVector() {
+ this(VectorizedRowBatch.DEFAULT_SIZE);
+ }
+
+ /**
+ * Don't use this except for testing purposes.
+ *
+ * @param len the number of rows
+ */
+ public TimestampColumnVector(int len) {
+ super(Type.TIMESTAMP, len);
+
+ time = new long[len];
+ nanos = new int[len];
+
+ scratchTimestamp = new Timestamp(0);
+
+ scratchWritable = null; // Allocated by caller.
+ }
+
+ /**
+ * Return the number of rows.
+ * @return
+ */
+ public int getLength() {
+ return time.length;
+ }
+
+ /**
+ * Return a row's Timestamp.getTime() value.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @return
+ */
+ public long getTime(int elementNum) {
+ return time[elementNum];
+ }
+
+ /**
+ * Return a row's Timestamp.getNanos() value.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @return
+ */
+ public int getNanos(int elementNum) {
+ return nanos[elementNum];
+ }
+
+ /**
+ * Set a Timestamp object from a row of the column.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param timestamp
+ * @param elementNum
+ */
+ public void timestampUpdate(Timestamp timestamp, int elementNum) {
+ timestamp.setTime(time[elementNum]);
+ timestamp.setNanos(nanos[elementNum]);
+ }
+
+ /**
+ * Return the scratch Timestamp object set from a row.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @return
+ */
+ public Timestamp asScratchTimestamp(int elementNum) {
+ scratchTimestamp.setTime(time[elementNum]);
+ scratchTimestamp.setNanos(nanos[elementNum]);
+ return scratchTimestamp;
+ }
+
+ /**
+ * Return the scratch timestamp (contents undefined).
+ * @return
+ */
+ public Timestamp getScratchTimestamp() {
+ return scratchTimestamp;
+ }
+
+ /**
+ * Return a long representation of a Timestamp.
+ * @param elementNum
+ * @return
+ */
+ public long getTimestampAsLong(int elementNum) {
+ scratchTimestamp.setTime(time[elementNum]);
+ scratchTimestamp.setNanos(nanos[elementNum]);
+ return getTimestampAsLong(scratchTimestamp);
+ }
+
+ /**
+ * Return a long representation of a Timestamp.
+ * @param timestamp
+ * @return
+ */
+ public static long getTimestampAsLong(Timestamp timestamp) {
+ return millisToSeconds(timestamp.getTime());
+ }
+
+ // Copy of TimestampWritable.millisToSeconds
+ /**
+ * Rounds the number of milliseconds relative to the epoch down to the nearest whole number of
+ * seconds. 500 would round to 0, -500 would round to -1.
+ */
+ private static long millisToSeconds(long millis) {
+ if (millis >= 0) {
+ return millis / 1000;
+ } else {
+ return (millis - 999) / 1000;
+ }
+ }
+
+ /**
+ * Return a double representation of a Timestamp.
+ * @param elementNum
+ * @return
+ */
+ public double getDouble(int elementNum) {
+ scratchTimestamp.setTime(time[elementNum]);
+ scratchTimestamp.setNanos(nanos[elementNum]);
+ return getDouble(scratchTimestamp);
+ }
+
+ /**
+ * Return a double representation of a Timestamp.
+ * @param timestamp
+ * @return
+ */
+ public static double getDouble(Timestamp timestamp) {
+ // Same algorithm as TimestampWritable (not currently import-able here).
+ double seconds, nanos;
+ seconds = millisToSeconds(timestamp.getTime());
+ nanos = timestamp.getNanos();
+ return seconds + nanos / 1000000000;
+ }
+
+ /**
+ * Compare row to Timestamp.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @param timestamp
+ * @return -1, 0, 1 standard compareTo values.
+ */
+ public int compareTo(int elementNum, Timestamp timestamp) {
+ return asScratchTimestamp(elementNum).compareTo(timestamp);
+ }
+
+ /**
+ * Compare Timestamp to row.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param timestamp
+ * @param elementNum
+ * @return -1, 0, 1 standard compareTo values.
+ */
+ public int compareTo(Timestamp timestamp, int elementNum) {
+ return timestamp.compareTo(asScratchTimestamp(elementNum));
+ }
+
+ /**
+ * Compare a row to another TimestampColumnVector's row.
+ * @param elementNum1
+ * @param timestampColVector2
+ * @param elementNum2
+ * @return
+ */
+ public int compareTo(int elementNum1, TimestampColumnVector timestampColVector2,
+ int elementNum2) {
+ return asScratchTimestamp(elementNum1).compareTo(
+ timestampColVector2.asScratchTimestamp(elementNum2));
+ }
+
+ /**
+ * Compare another TimestampColumnVector's row to a row.
+ * @param timestampColVector1
+ * @param elementNum1
+ * @param elementNum2
+ * @return
+ */
+ public int compareTo(TimestampColumnVector timestampColVector1, int elementNum1,
+ int elementNum2) {
+ return timestampColVector1.asScratchTimestamp(elementNum1).compareTo(
+ asScratchTimestamp(elementNum2));
+ }
+
+ @Override
+ public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) {
+
+ TimestampColumnVector timestampColVector = (TimestampColumnVector) inputVector;
+
+ time[outElementNum] = timestampColVector.time[inputElementNum];
+ nanos[outElementNum] = timestampColVector.nanos[inputElementNum];
+ }
+
+ // Simplify vector by brute-force flattening noNulls and isRepeating
+ // This can be used to reduce combinatorial explosion of code paths in VectorExpressions
+ // with many arguments.
+ public void flatten(boolean selectedInUse, int[] sel, int size) {
+ flattenPush();
+ if (isRepeating) {
+ isRepeating = false;
+ long repeatFastTime = time[0];
+ int repeatNanos = nanos[0];
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ time[i] = repeatFastTime;
+ nanos[i] = repeatNanos;
+ }
+ } else {
+ Arrays.fill(time, 0, size, repeatFastTime);
+ Arrays.fill(nanos, 0, size, repeatNanos);
+ }
+ flattenRepeatingNulls(selectedInUse, sel, size);
+ }
+ flattenNoNulls(selectedInUse, sel, size);
+ }
+
+ /**
+ * Set a row from a timestamp.
+ * We assume the entry has already been isRepeated adjusted.
+ * @param elementNum
+ * @param timestamp
+ */
+ public void set(int elementNum, Timestamp timestamp) {
+ if (timestamp == null) {
+ this.noNulls = false;
+ this.isNull[elementNum] = true;
+ } else {
+ this.time[elementNum] = timestamp.getTime();
+ this.nanos[elementNum] = timestamp.getNanos();
+ }
+ }
+
+ /**
+ * Set a row from the current value in the scratch timestamp.
+ * @param elementNum
+ */
+ public void setFromScratchTimestamp(int elementNum) {
+ this.time[elementNum] = scratchTimestamp.getTime();
+ this.nanos[elementNum] = scratchTimestamp.getNanos();
+ }
+
+ /**
+ * Set row to standard null value(s).
+ * We assume the entry has already been isRepeated adjusted.
+ * @param elementNum
+ */
+ public void setNullValue(int elementNum) {
+ time[elementNum] = 0;
+ nanos[elementNum] = 1;
+ }
+
+ // Copy the current object contents into the output. Only copy selected entries,
+ // as indicated by selectedInUse and the sel array.
+ public void copySelected(
+ boolean selectedInUse, int[] sel, int size, TimestampColumnVector output) {
+
+ // Output has nulls if and only if input has nulls.
+ output.noNulls = noNulls;
+ output.isRepeating = false;
+
+ // Handle repeating case
+ if (isRepeating) {
+ output.time[0] = time[0];
+ output.nanos[0] = nanos[0];
+ output.isNull[0] = isNull[0];
+ output.isRepeating = true;
+ return;
+ }
+
+ // Handle normal case
+
+ // Copy data values over
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.time[i] = time[i];
+ output.nanos[i] = nanos[i];
+ }
+ }
+ else {
+ System.arraycopy(time, 0, output.time, 0, size);
+ System.arraycopy(nanos, 0, output.nanos, 0, size);
+ }
+
+ // Copy nulls over if needed
+ if (!noNulls) {
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.isNull[i] = isNull[i];
+ }
+ }
+ else {
+ System.arraycopy(isNull, 0, output.isNull, 0, size);
+ }
+ }
+ }
+
+ /**
+ * Fill all the vector entries with a timestamp.
+ * @param timestamp
+ */
+ public void fill(Timestamp timestamp) {
+ noNulls = true;
+ isRepeating = true;
+ time[0] = timestamp.getTime();
+ nanos[0] = timestamp.getNanos();
+ }
+
+ /**
+ * Return a convenience writable object stored by this column vector.
+ * Supports keeping a TimestampWritable object without having to import that definition...
+ * @return
+ */
+ public Writable getScratchWritable() {
+ return scratchWritable;
+ }
+
+ /**
+ * Set the convenience writable object stored by this column vector
+ * @param scratchWritable
+ */
+ public void setScratchWritable(Writable scratchWritable) {
+ this.scratchWritable = scratchWritable;
+ }
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ scratchTimestamp.setTime(time[row]);
+ scratchTimestamp.setNanos(nanos[row]);
+ buffer.append(scratchTimestamp.toString());
+ } else {
+ buffer.append("null");
+ }
+ }
+
+ @Override
+ public void ensureSize(int size, boolean preserveData) {
+ super.ensureSize(size, preserveData);
+ if (size <= time.length) return;
+ long[] oldTime = time;
+ int[] oldNanos = nanos;
+ time = new long[size];
+ nanos = new int[size];
+ if (preserveData) {
+ if (isRepeating) {
+ time[0] = oldTime[0];
+ nanos[0] = oldNanos[0];
+ } else {
+ System.arraycopy(oldTime, 0, time, 0, oldTime.length);
+ System.arraycopy(oldNanos, 0, nanos, 0, oldNanos.length);
+ }
+ }
+ }
+
+ @Override
+ public void shallowCopyTo(ColumnVector otherCv) {
+ TimestampColumnVector other = (TimestampColumnVector)otherCv;
+ super.shallowCopyTo(other);
+ other.time = time;
+ other.nanos = nanos;
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java
new file mode 100644
index 0000000000000000000000000000000000000000..e844b11f9634255f121ab4225e227ef7801adb2e
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+/**
+ * The representation of a vectorized column of struct objects.
+ *
+ * Each field is represented by a separate inner ColumnVector. Since this
+ * ColumnVector doesn't own any per row data other that the isNull flag, the
+ * isRepeating only covers the isNull array.
+ */
+public class UnionColumnVector extends ColumnVector {
+
+ public int[] tags;
+ public ColumnVector[] fields;
+
+ public UnionColumnVector() {
+ this(VectorizedRowBatch.DEFAULT_SIZE);
+ }
+
+ /**
+ * Constructor for UnionColumnVector
+ *
+ * @param len Vector length
+ * @param fields the field column vectors
+ */
+ public UnionColumnVector(int len, ColumnVector... fields) {
+ super(Type.UNION, len);
+ tags = new int[len];
+ this.fields = fields;
+ }
+
+ @Override
+ public void flatten(boolean selectedInUse, int[] sel, int size) {
+ flattenPush();
+ for(int i=0; i < fields.length; ++i) {
+ fields[i].flatten(selectedInUse, sel, size);
+ }
+ flattenNoNulls(selectedInUse, sel, size);
+ }
+
+ @Override
+ public void setElement(int outElementNum, int inputElementNum,
+ ColumnVector inputVector) {
+ if (inputVector.isRepeating) {
+ inputElementNum = 0;
+ }
+ if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) {
+ isNull[outElementNum] = false;
+ UnionColumnVector input = (UnionColumnVector) inputVector;
+ tags[outElementNum] = input.tags[inputElementNum];
+ fields[tags[outElementNum]].setElement(outElementNum, inputElementNum,
+ input.fields[tags[outElementNum]]);
+ } else {
+ noNulls = false;
+ isNull[outElementNum] = true;
+ }
+ }
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ buffer.append("{\"tag\": ");
+ buffer.append(tags[row]);
+ buffer.append(", \"value\": ");
+ fields[tags[row]].stringifyValue(buffer, row);
+ buffer.append('}');
+ } else {
+ buffer.append("null");
+ }
+ }
+
+ @Override
+ public void ensureSize(int size, boolean preserveData) {
+ super.ensureSize(size, preserveData);
+ if (tags.length < size) {
+ if (preserveData) {
+ int[] oldTags = tags;
+ tags = new int[size];
+ System.arraycopy(oldTags, 0, tags, 0, oldTags.length);
+ } else {
+ tags = new int[size];
+ }
+ for(int i=0; i < fields.length; ++i) {
+ fields[i].ensureSize(size, preserveData);
+ }
+ }
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ for(int i =0; i < fields.length; ++i) {
+ fields[i].reset();
+ }
+ }
+
+ @Override
+ public void init() {
+ super.init();
+ for(int i =0; i < fields.length; ++i) {
+ fields[i].init();
+ }
+ }
+
+ @Override
+ public void unFlatten() {
+ super.unFlatten();
+ for(int i=0; i < fields.length; ++i) {
+ fields[i].unFlatten();
+ }
+ }
+
+ @Override
+ public void setRepeating(boolean isRepeating) {
+ super.setRepeating(isRepeating);
+ for(int i=0; i < fields.length; ++i) {
+ fields[i].setRepeating(isRepeating);
+ }
+ }
+
+ @Override
+ public void shallowCopyTo(ColumnVector otherCv) {
+ throw new UnsupportedOperationException(); // Implement if needed.
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
new file mode 100644
index 0000000000000000000000000000000000000000..5e5f13ddc7064aa9343376320e50f2da2c3b20ac
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
@@ -0,0 +1,264 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * A VectorizedRowBatch is a set of rows, organized with each column
+ * as a vector. It is the unit of query execution, organized to minimize
+ * the cost per row and achieve high cycles-per-instruction.
+ * The major fields are public by design to allow fast and convenient
+ * access by the vectorized query execution code.
+ */
+public class VectorizedRowBatch implements Writable {
+ public int numCols; // number of columns
+ public ColumnVector[] cols; // a vector for each column
+ public int size; // number of rows that qualify (i.e. haven't been filtered out)
+ public int[] selected; // array of positions of selected values
+ public int[] projectedColumns;
+ public int projectionSize;
+
+ private int dataColumnCount;
+ private int partitionColumnCount;
+
+
+ /*
+ * If no filtering has been applied yet, selectedInUse is false,
+ * meaning that all rows qualify. If it is true, then the selected[] array
+ * records the offsets of qualifying rows.
+ */
+ public boolean selectedInUse;
+
+ // If this is true, then there is no data in the batch -- we have hit the end of input.
+ public boolean endOfFile;
+
+ /*
+ * This number is carefully chosen to minimize overhead and typically allows
+ * one VectorizedRowBatch to fit in cache.
+ */
+ public static final int DEFAULT_SIZE = 1024;
+
+ /*
+ * This number is a safety limit for 32MB of writables.
+ */
+ public static final int DEFAULT_BYTES = 32 * 1024 * 1024;
+
+ /**
+ * Return a batch with the specified number of columns.
+ * This is the standard constructor -- all batches should be the same size
+ *
+ * @param numCols the number of columns to include in the batch
+ */
+ public VectorizedRowBatch(int numCols) {
+ this(numCols, DEFAULT_SIZE);
+ }
+
+ /**
+ * Return a batch with the specified number of columns and rows.
+ * Only call this constructor directly for testing purposes.
+ * Batch size should normally always be defaultSize.
+ *
+ * @param numCols the number of columns to include in the batch
+ * @param size the number of rows to include in the batch
+ */
+ public VectorizedRowBatch(int numCols, int size) {
+ this.numCols = numCols;
+ this.size = size;
+ selected = new int[size];
+ selectedInUse = false;
+ this.cols = new ColumnVector[numCols];
+ projectedColumns = new int[numCols];
+
+ // Initially all columns are projected and in the same order
+ projectionSize = numCols;
+ for (int i = 0; i < numCols; i++) {
+ projectedColumns[i] = i;
+ }
+
+ dataColumnCount = -1;
+ partitionColumnCount = -1;
+ }
+
+ public void setPartitionInfo(int dataColumnCount, int partitionColumnCount) {
+ this.dataColumnCount = dataColumnCount;
+ this.partitionColumnCount = partitionColumnCount;
+ }
+
+ public int getDataColumnCount() {
+ return dataColumnCount;
+ }
+
+ public int getPartitionColumnCount() {
+ return partitionColumnCount;
+ }
+
+ /**
+ * Returns the maximum size of the batch (number of rows it can hold)
+ */
+ public int getMaxSize() {
+ return selected.length;
+ }
+
+ /**
+ * Return count of qualifying rows.
+ *
+ * @return number of rows that have not been filtered out
+ */
+ public long count() {
+ return size;
+ }
+
+ private static String toUTF8(Object o) {
+ if(o == null || o instanceof NullWritable) {
+ return "\\N"; /* as found in LazySimpleSerDe's nullSequence */
+ }
+ return o.toString();
+ }
+
+ @Override
+ public String toString() {
+ if (size == 0) {
+ return "";
+ }
+ StringBuilder b = new StringBuilder();
+ b.append("Column vector types: ");
+ for (int k = 0; k < projectionSize; k++) {
+ int projIndex = projectedColumns[k];
+ ColumnVector cv = cols[projIndex];
+ if (k > 0) {
+ b.append(", ");
+ }
+ b.append(projIndex);
+ b.append(":");
+ String colVectorType = null;
+ if (cv instanceof LongColumnVector) {
+ colVectorType = "LONG";
+ } else if (cv instanceof DoubleColumnVector) {
+ colVectorType = "DOUBLE";
+ } else if (cv instanceof BytesColumnVector) {
+ colVectorType = "BYTES";
+ } else if (cv instanceof DecimalColumnVector) {
+ colVectorType = "DECIMAL";
+ } else if (cv instanceof TimestampColumnVector) {
+ colVectorType = "TIMESTAMP";
+ } else if (cv instanceof IntervalDayTimeColumnVector) {
+ colVectorType = "INTERVAL_DAY_TIME";
+ } else if (cv instanceof ListColumnVector) {
+ colVectorType = "LIST";
+ } else if (cv instanceof MapColumnVector) {
+ colVectorType = "MAP";
+ } else if (cv instanceof StructColumnVector) {
+ colVectorType = "STRUCT";
+ } else if (cv instanceof UnionColumnVector) {
+ colVectorType = "UNION";
+ } else {
+ colVectorType = "Unknown";
+ }
+ b.append(colVectorType);
+ }
+ b.append('\n');
+
+ if (this.selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = selected[j];
+ b.append('[');
+ for (int k = 0; k < projectionSize; k++) {
+ int projIndex = projectedColumns[k];
+ ColumnVector cv = cols[projIndex];
+ if (k > 0) {
+ b.append(", ");
+ }
+ cv.stringifyValue(b, i);
+ }
+ b.append(']');
+ if (j < size - 1) {
+ b.append('\n');
+ }
+ }
+ } else {
+ for (int i = 0; i < size; i++) {
+ b.append('[');
+ for (int k = 0; k < projectionSize; k++) {
+ int projIndex = projectedColumns[k];
+ ColumnVector cv = cols[projIndex];
+ if (k > 0) {
+ b.append(", ");
+ }
+ if (cv != null) {
+ try {
+ cv.stringifyValue(b, i);
+ } catch (Exception ex) {
+ b.append("");
+ }
+ }
+ }
+ b.append(']');
+ if (i < size - 1) {
+ b.append('\n');
+ }
+ }
+ }
+ return b.toString();
+ }
+
+ @Override
+ public void readFields(DataInput arg0) throws IOException {
+ throw new UnsupportedOperationException("Do you really need me?");
+ }
+
+ @Override
+ public void write(DataOutput arg0) throws IOException {
+ throw new UnsupportedOperationException("Don't call me");
+ }
+
+ /**
+ * Resets the row batch to default state
+ * - sets selectedInUse to false
+ * - sets size to 0
+ * - sets endOfFile to false
+ * - resets each column
+ * - inits each column
+ */
+ public void reset() {
+ selectedInUse = false;
+ size = 0;
+ endOfFile = false;
+ for (ColumnVector vc : cols) {
+ if (vc != null) {
+ vc.reset();
+ vc.init();
+ }
+ }
+ }
+
+ /**
+ * Set the maximum number of rows in the batch.
+ * Data is not preserved.
+ */
+ public void ensureSize(int rows) {
+ for(int i=0; i < cols.length; ++i) {
+ cols[i].ensureSize(rows, false);
+ }
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java
new file mode 100644
index 0000000000000000000000000000000000000000..f2ae9bce709850345307c2068c49bfeb38dd9c44
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringExpr.java
@@ -0,0 +1,414 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import java.util.Arrays;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+
+/**
+ * String expression evaluation helper functions.
+ */
+public class StringExpr {
+
+ /* Compare two strings from two byte arrays each
+ * with their own start position and length.
+ * Use lexicographic unsigned byte value order.
+ * This is what's used for UTF-8 sort order.
+ * Return negative value if arg1 < arg2, 0 if arg1 = arg2,
+ * positive if arg1 > arg2.
+ */
+ public static int compare(byte[] arg1, int start1, int len1, byte[] arg2, int start2, int len2) {
+ for (int i = 0; i < len1 && i < len2; i++) {
+ // Note the "& 0xff" is just a way to convert unsigned bytes to signed integer.
+ int b1 = arg1[i + start1] & 0xff;
+ int b2 = arg2[i + start2] & 0xff;
+ if (b1 != b2) {
+ return b1 - b2;
+ }
+ }
+ return len1 - len2;
+ }
+
+ /* Determine if two strings are equal from two byte arrays each
+ * with their own start position and length.
+ * Use lexicographic unsigned byte value order.
+ * This is what's used for UTF-8 sort order.
+ */
+ public static boolean equal(byte[] arg1, final int start1, final int len1,
+ byte[] arg2, final int start2, final int len2) {
+ if (len1 != len2) {
+ return false;
+ }
+ if (len1 == 0) {
+ return true;
+ }
+
+ // do bounds check for OOB exception
+ if (arg1[start1] != arg2[start2]
+ || arg1[start1 + len1 - 1] != arg2[start2 + len2 - 1]) {
+ return false;
+ }
+
+ if (len1 == len2) {
+ // prove invariant to the compiler: len1 = len2
+ // all array access between (start1, start1+len1)
+ // and (start2, start2+len2) are valid
+ // no more OOB exceptions are possible
+ final int step = 8;
+ final int remainder = len1 % step;
+ final int wlen = len1 - remainder;
+ // suffix first
+ for (int i = wlen; i < len1; i++) {
+ if (arg1[start1 + i] != arg2[start2 + i]) {
+ return false;
+ }
+ }
+ // SIMD loop
+ for (int i = 0; i < wlen; i += step) {
+ final int s1 = start1 + i;
+ final int s2 = start2 + i;
+ boolean neq = false;
+ for (int j = 0; j < step; j++) {
+ neq = (arg1[s1 + j] != arg2[s2 + j]) || neq;
+ }
+ if (neq) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+ }
+
+ public static int characterCount(byte[] bytes) {
+ int end = bytes.length;
+
+ // count characters
+ int j = 0;
+ int charCount = 0;
+ while(j < end) {
+ // UTF-8 continuation bytes have 2 high bits equal to 0x80.
+ if ((bytes[j] & 0xc0) != 0x80) {
+ ++charCount;
+ }
+ j++;
+ }
+ return charCount;
+ }
+
+ public static int characterCount(byte[] bytes, int start, int length) {
+ int end = start + length;
+
+ // count characters
+ int j = start;
+ int charCount = 0;
+ while(j < end) {
+ // UTF-8 continuation bytes have 2 high bits equal to 0x80.
+ if ((bytes[j] & 0xc0) != 0x80) {
+ ++charCount;
+ }
+ j++;
+ }
+ return charCount;
+ }
+
+ // A setVal with the same function signature as rightTrim, leftTrim, truncate, etc, below.
+ // Useful for class generation via templates.
+ public static void assign(BytesColumnVector outV, int i, byte[] bytes, int start, int length) {
+ // set output vector
+ outV.setVal(i, bytes, start, length);
+ }
+
+ /*
+ * Right trim a slice of a byte array and return the new byte length.
+ */
+ public static int rightTrim(byte[] bytes, int start, int length) {
+ // skip trailing blank characters
+ int j = start + length - 1;
+ while(j >= start && bytes[j] == 0x20) {
+ j--;
+ }
+
+ return (j - start) + 1;
+ }
+
+ /*
+ * Right trim a slice of a byte array and place the result into element i of a vector.
+ */
+ public static void rightTrim(BytesColumnVector outV, int i, byte[] bytes, int start, int length) {
+ // skip trailing blank characters
+ int j = start + length - 1;
+ while(j >= start && bytes[j] == 0x20) {
+ j--;
+ }
+
+ // set output vector
+ outV.setVal(i, bytes, start, (j - start) + 1);
+ }
+
+ /*
+ * Truncate a slice of a byte array to a maximum number of characters and
+ * return the new byte length.
+ */
+ public static int truncate(byte[] bytes, int start, int length, int maxLength) {
+ int end = start + length;
+
+ // count characters forward
+ int j = start;
+ int charCount = 0;
+ while(j < end) {
+ // UTF-8 continuation bytes have 2 high bits equal to 0x80.
+ if ((bytes[j] & 0xc0) != 0x80) {
+ if (charCount == maxLength) {
+ break;
+ }
+ ++charCount;
+ }
+ j++;
+ }
+ return (j - start);
+ }
+
+ /*
+ * Truncate a slice of a byte array to a maximum number of characters and
+ * place the result into element i of a vector.
+ */
+ public static void truncate(BytesColumnVector outV, int i, byte[] bytes, int start, int length, int maxLength) {
+ int end = start + length;
+
+ // count characters forward
+ int j = start;
+ int charCount = 0;
+ while(j < end) {
+ // UTF-8 continuation bytes have 2 high bits equal to 0x80.
+ if ((bytes[j] & 0xc0) != 0x80) {
+ if (charCount == maxLength) {
+ break;
+ }
+ ++charCount;
+ }
+ j++;
+ }
+
+ // set output vector
+ outV.setVal(i, bytes, start, (j - start));
+ }
+
+ /*
+ * Truncate a byte array to a maximum number of characters and
+ * return a byte array with only truncated bytes.
+ */
+ public static byte[] truncateScalar(byte[] bytes, int maxLength) {
+ int end = bytes.length;
+
+ // count characters forward
+ int j = 0;
+ int charCount = 0;
+ while(j < end) {
+ // UTF-8 continuation bytes have 2 high bits equal to 0x80.
+ if ((bytes[j] & 0xc0) != 0x80) {
+ if (charCount == maxLength) {
+ break;
+ }
+ ++charCount;
+ }
+ j++;
+ }
+ if (j == end) {
+ return bytes;
+ } else {
+ return Arrays.copyOf(bytes, j);
+ }
+ }
+
+ /*
+ * Right trim and truncate a slice of a byte array to a maximum number of characters and
+ * return the new byte length.
+ */
+ public static int rightTrimAndTruncate(byte[] bytes, int start, int length, int maxLength) {
+ int end = start + length;
+
+ // count characters forward and watch for final run of pads
+ int j = start;
+ int charCount = 0;
+ int padRunStart = -1;
+ while(j < end) {
+ // UTF-8 continuation bytes have 2 high bits equal to 0x80.
+ if ((bytes[j] & 0xc0) != 0x80) {
+ if (charCount == maxLength) {
+ break;
+ }
+ if (bytes[j] == 0x20) {
+ if (padRunStart == -1) {
+ padRunStart = j;
+ }
+ } else {
+ padRunStart = -1;
+ }
+ ++charCount;
+ } else {
+ padRunStart = -1;
+ }
+ j++;
+ }
+ if (padRunStart != -1) {
+ return (padRunStart - start);
+ } else {
+ return (j - start);
+ }
+ }
+
+ /*
+ * Right trim and truncate a slice of a byte array to a maximum number of characters and
+ * place the result into element i of a vector.
+ */
+ public static void rightTrimAndTruncate(BytesColumnVector outV, int i, byte[] bytes, int start, int length, int maxLength) {
+ int end = start + length;
+
+ // count characters forward and watch for final run of pads
+ int j = start;
+ int charCount = 0;
+ int padRunStart = -1;
+ while(j < end) {
+ // UTF-8 continuation bytes have 2 high bits equal to 0x80.
+ if ((bytes[j] & 0xc0) != 0x80) {
+ if (charCount == maxLength) {
+ break;
+ }
+ if (bytes[j] == 0x20) {
+ if (padRunStart == -1) {
+ padRunStart = j;
+ }
+ } else {
+ padRunStart = -1;
+ }
+ ++charCount;
+ } else {
+ padRunStart = -1;
+ }
+ j++;
+ }
+ // set output vector
+ if (padRunStart != -1) {
+ outV.setVal(i, bytes, start, (padRunStart - start));
+ } else {
+ outV.setVal(i, bytes, start, (j - start) );
+ }
+ }
+
+ /*
+ * Right trim and truncate a byte array to a maximum number of characters and
+ * return a byte array with only the trimmed and truncated bytes.
+ */
+ public static byte[] rightTrimAndTruncateScalar(byte[] bytes, int maxLength) {
+ int end = bytes.length;
+
+ // count characters forward and watch for final run of pads
+ int j = 0;
+ int charCount = 0;
+ int padRunStart = -1;
+ while(j < end) {
+ // UTF-8 continuation bytes have 2 high bits equal to 0x80.
+ if ((bytes[j] & 0xc0) != 0x80) {
+ if (charCount == maxLength) {
+ break;
+ }
+ if (bytes[j] == 0x20) {
+ if (padRunStart == -1) {
+ padRunStart = j;
+ }
+ } else {
+ padRunStart = -1;
+ }
+ ++charCount;
+ } else {
+ padRunStart = -1;
+ }
+ j++;
+ }
+ if (padRunStart != -1) {
+ return Arrays.copyOf(bytes, padRunStart);
+ } else if (j == end) {
+ return bytes;
+ } else {
+ return Arrays.copyOf(bytes, j);
+ }
+ }
+
+ /*
+ * Compiles the given pattern with a proper algorithm.
+ */
+ public static Finder compile(byte[] pattern) {
+ return new BoyerMooreHorspool(pattern);
+ }
+
+ /*
+ * A finder finds the first index of its pattern in a given byte array.
+ * Its thread-safety depends on its implementation.
+ */
+ public interface Finder {
+ int find(byte[] input, int start, int len);
+ }
+
+ /*
+ * StringExpr uses Boyer Moore Horspool algorithm to find faster.
+ * It is thread-safe, because it holds final member instances only.
+ * See https://en.wikipedia.org/wiki/Boyer–Moore–Horspool_algorithm .
+ */
+ private static class BoyerMooreHorspool implements Finder {
+ private static final int MAX_BYTE = 0xff;
+ private final long[] shift = new long[MAX_BYTE];
+ private final byte[] pattern;
+ private final int plen;
+
+ public BoyerMooreHorspool(byte[] pattern) {
+ this.pattern = pattern;
+ this.plen = pattern.length;
+ Arrays.fill(shift, plen);
+ for (int i = 0; i < plen - 1; i++) {
+ shift[pattern[i] & MAX_BYTE] = plen - i - 1;
+ }
+ }
+
+ public int find(byte[] input, int start, int len) {
+ if (pattern.length == 0) {
+ return 0;
+ }
+
+ final int end = start + len;
+ int next = start + plen - 1;
+ final int plen = this.plen;
+ final byte[] pattern = this.pattern;
+ while (next < end) {
+ int s_tmp = next;
+ int p_tmp = plen - 1;
+ while (input[s_tmp] == pattern[p_tmp]) {
+ p_tmp--;
+ if (p_tmp < 0) {
+ return s_tmp;
+ }
+ s_tmp--;
+ }
+ next += shift[input[next] & MAX_BYTE];
+ }
+ return -1;
+ }
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/ExpressionTree.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/ExpressionTree.java
new file mode 100644
index 0000000000000000000000000000000000000000..443083d228502f7c78f3bd4ece320c38812a6c68
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/ExpressionTree.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.sarg;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * The inner representation of the SearchArgument. Most users should not
+ * need this interface, it is only for file formats that need to translate
+ * the SearchArgument into an internal form.
+ */
+public class ExpressionTree {
+ public enum Operator {OR, AND, NOT, LEAF, CONSTANT}
+ private final Operator operator;
+ private final List children;
+ private int leaf;
+ private final SearchArgument.TruthValue constant;
+
+ ExpressionTree() {
+ operator = null;
+ children = null;
+ leaf = 0;
+ constant = null;
+ }
+
+ ExpressionTree(Operator op, ExpressionTree... kids) {
+ operator = op;
+ children = new ArrayList();
+ leaf = -1;
+ this.constant = null;
+ Collections.addAll(children, kids);
+ }
+
+ ExpressionTree(int leaf) {
+ operator = Operator.LEAF;
+ children = null;
+ this.leaf = leaf;
+ this.constant = null;
+ }
+
+ ExpressionTree(SearchArgument.TruthValue constant) {
+ operator = Operator.CONSTANT;
+ children = null;
+ this.leaf = -1;
+ this.constant = constant;
+ }
+
+ ExpressionTree(ExpressionTree other) {
+ this.operator = other.operator;
+ if (other.children == null) {
+ this.children = null;
+ } else {
+ this.children = new ArrayList();
+ for(ExpressionTree child: other.children) {
+ children.add(new ExpressionTree(child));
+ }
+ }
+ this.leaf = other.leaf;
+ this.constant = other.constant;
+ }
+
+ public SearchArgument.TruthValue evaluate(SearchArgument.TruthValue[] leaves
+ ) {
+ SearchArgument.TruthValue result = null;
+ switch (operator) {
+ case OR:
+ for(ExpressionTree child: children) {
+ result = child.evaluate(leaves).or(result);
+ }
+ return result;
+ case AND:
+ for(ExpressionTree child: children) {
+ result = child.evaluate(leaves).and(result);
+ }
+ return result;
+ case NOT:
+ return children.get(0).evaluate(leaves).not();
+ case LEAF:
+ return leaves[leaf];
+ case CONSTANT:
+ return constant;
+ default:
+ throw new IllegalStateException("Unknown operator: " + operator);
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buffer = new StringBuilder();
+ switch (operator) {
+ case OR:
+ buffer.append("(or");
+ for(ExpressionTree child: children) {
+ buffer.append(' ');
+ buffer.append(child.toString());
+ }
+ buffer.append(')');
+ break;
+ case AND:
+ buffer.append("(and");
+ for(ExpressionTree child: children) {
+ buffer.append(' ');
+ buffer.append(child.toString());
+ }
+ buffer.append(')');
+ break;
+ case NOT:
+ buffer.append("(not ");
+ buffer.append(children.get(0));
+ buffer.append(')');
+ break;
+ case LEAF:
+ buffer.append("leaf-");
+ buffer.append(leaf);
+ break;
+ case CONSTANT:
+ buffer.append(constant);
+ break;
+ }
+ return buffer.toString();
+ }
+
+ public Operator getOperator() {
+ return operator;
+ }
+
+ public List getChildren() {
+ return children;
+ }
+
+ public SearchArgument.TruthValue getConstant() {
+ return constant;
+ }
+
+ public int getLeaf() {
+ return leaf;
+ }
+
+ public void setLeaf(int leaf) {
+ this.leaf = leaf;
+ }
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/LiteralDelegate.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/LiteralDelegate.java
new file mode 100644
index 0000000000000000000000000000000000000000..bd8a5ce57860d20f08c9012a2332c7dcdec9fee8
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/LiteralDelegate.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.sarg;
+
+import org.apache.hadoop.conf.Configurable;
+
+/**
+ * Interface to retrieve a literal value
+ */
+public interface LiteralDelegate extends Configurable {
+
+ Object getLiteral();
+
+ String getId();
+}
diff --git a/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java
new file mode 100644
index 0000000000000000000000000000000000000000..469a3dad47c79e51d24115db469647596e7b9785
--- /dev/null
+++ b/storage-api/hive-storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.sarg;
+
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+
+import java.sql.Date;
+import java.sql.Timestamp;
+import java.util.List;
+
+/**
+ * The primitive predicates that form a SearchArgument.
+ */
+public interface PredicateLeaf {
+
+ /**
+ * The possible operators for predicates. To get the opposites, construct
+ * an expression with a not operator.
+ */
+ public static enum Operator {
+ EQUALS,
+ NULL_SAFE_EQUALS,
+ LESS_THAN,
+ LESS_THAN_EQUALS,
+ IN,
+ BETWEEN,
+ IS_NULL
+ }
+
+ /**
+ * The possible types for sargs.
+ */
+ public static enum Type {
+ LONG(Long.class), // all of the integer types
+ FLOAT(Double.class), // float and double
+ STRING(String.class), // string, char, varchar
+ DATE(Date.class),
+ DECIMAL(HiveDecimalWritable.class),
+ TIMESTAMP(Timestamp.class),
+ BOOLEAN(Boolean.class);
+
+ private final Class cls;
+ Type(Class cls) {
+ this.cls = cls;
+ }
+
+ /**
+ * For all SARG leaves, the values must be the matching class.
+ * @return the value class
+ */
+ public Class getValueClass() {
+ return cls;
+ }
+ }
+
+ /**
+ * Get the operator for the leaf.
+ */
+ public Operator getOperator();
+
+ /**
+ * Get the type of the column and literal by the file format.
+ */
+ public Type getType();
+
+ /**
+ * Get the simple column name.
+ * @return the column name
+ */
+ public String getColumnName();
+
+ /**
+ * Get the literal half of the predicate leaf. Adapt the original type for what orc needs
+ *
+ * @return an Integer, Long, Double, or String
+ */
+ public Object getLiteral();
+
+ /**
+ * For operators with multiple literals (IN and BETWEEN), get the literals.
+ *
+ * @return the list of literals (Integer, Longs, Doubles, or Strings)
+ *
+ */
+ public List