From c799a52aeab95542aa5d9eb55dc3987af26f2878 Mon Sep 17 00:00:00 2001 From: Alan Gates Date: Wed, 2 Aug 2017 11:38:27 -0700 Subject: [PATCH 1/2] Removed use of TypeInfo from metastore classes. Added metastore specific type information. --- .../hadoop/hive/metastore/HiveAlterHandler.java | 24 +- .../hadoop/hive/metastore/MetaStoreDirectSql.java | 7 +- .../apache/hadoop/hive/metastore/ObjectStore.java | 11 +- .../hive/metastore/PartitionExpressionProxy.java | 10 +- .../hadoop/hive/metastore/cache/CachedStore.java | 12 +- .../hive/metastore/parser/ExpressionTree.java | 6 +- .../MockPartitionExpressionForMetastore.java | 5 +- .../hadoop/hive/metastore/TestObjectStore.java | 11 +- .../ppr/PartitionExpressionForMetastore.java | 14 +- .../apache/hadoop/hive/metastore/ColumnType.java | 242 +++++++++++++++++++++ 10 files changed, 294 insertions(+), 48 deletions(-) create mode 100644 standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 28c3cfed6d7..b279e1d5677 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -156,8 +156,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, false)) { // Throws InvalidOperationException if the new column types are not // compatible with the current column types. - MetaStoreUtils.throwExceptionIfIncompatibleColTypeChange( - oldt.getSd().getCols(), newt.getSd().getCols()); + checkColTypeChangeCompatible(oldt.getSd().getCols(), newt.getSd().getCols()); } //check that partition keys have not changed, except for virtual views @@ -852,4 +851,25 @@ private ColumnStatistics updateOrGetPartitionColumnStats( return newPartsColStats; } + + private void checkColTypeChangeCompatible(List oldCols, List newCols) + throws InvalidOperationException { + List incompatibleCols = new ArrayList<>(); + int maxCols = Math.min(oldCols.size(), newCols.size()); + for (int i = 0; i < maxCols; i++) { + if (!ColumnType.areColTypesCompatible( + ColumnType.getTypeName(oldCols.get(i).getType()), + ColumnType.getTypeName(newCols.get(i).getType()))) { + incompatibleCols.add(newCols.get(i).getName()); + } + } + if (!incompatibleCols.isEmpty()) { + throw new InvalidOperationException( + "The following columns have types incompatible with the existing " + + "columns in their respective positions :\n" + + org.apache.commons.lang.StringUtils.join(incompatibleCols, ',') + ); + } + } + } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 2dd7b68e228..b122c431b21 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -76,7 +76,6 @@ import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeVisitor; -import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hive.common.util.BloomFilter; import org.datanucleus.store.rdbms.query.ForwardQueryResult; import org.slf4j.Logger; @@ -1112,11 +1111,11 @@ protected boolean shouldStop() { Invalid; static FilterType fromType(String colTypeStr) { - if (colTypeStr.equals(serdeConstants.STRING_TYPE_NAME)) { + if (colTypeStr.equals(ColumnType.STRING_TYPE_NAME)) { return FilterType.String; - } else if (colTypeStr.equals(serdeConstants.DATE_TYPE_NAME)) { + } else if (colTypeStr.equals(ColumnType.DATE_TYPE_NAME)) { return FilterType.Date; - } else if (serdeConstants.IntegralTypes.contains(colTypeStr)) { + } else if (ColumnType.IntegralTypes.contains(colTypeStr)) { return FilterType.Integral; } return FilterType.Invalid; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 0a80241b771..a422b178255 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -160,8 +160,6 @@ import org.apache.hadoop.hive.metastore.parser.ExpressionTree; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.FilterBuilder; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.util.StringUtils; import org.apache.hive.common.util.HiveStringUtils; import org.apache.thrift.TException; @@ -2547,17 +2545,10 @@ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, String defaultPartName, short maxParts, List result) throws MetaException { result.addAll(getPartitionNamesNoTxn( table.getDbName(), table.getTableName(), maxParts)); - List columnNames = new ArrayList(); - List typeInfos = new ArrayList(); - for (FieldSchema fs : table.getPartitionKeys()) { - columnNames.add(fs.getName()); - typeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(fs.getType())); - } if (defaultPartName == null || defaultPartName.isEmpty()) { defaultPartName = HiveConf.getVar(getConf(), HiveConf.ConfVars.DEFAULTPARTITIONNAME); } - return expressionProxy.filterPartitionsByExpr( - columnNames, typeInfos, expr, defaultPartName, result); + return expressionProxy.filterPartitionsByExpr(table.getPartitionKeys(), expr, defaultPartName, result); } /** diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java b/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java index 8d6dc12f49e..af0a6bda543 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java @@ -20,10 +20,10 @@ import java.util.List; +import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; /** * The proxy interface that metastore uses for variety of QL operations (metastore can't depend @@ -41,16 +41,14 @@ /** * Filters the partition names via serialized Hive expression. - * @param partColumnNames Partition column names in the underlying table. - * @param partColumnTypeInfos Partition column types in the underlying table + * @param partColumns Partition columns in the underlying table. * @param expr Serialized expression. * @param defaultPartitionName Default partition name from job or server configuration. * @param partitionNames Partition names; the list is modified in place. * @return Whether there were any unknown partitions preserved in the name list. */ - boolean filterPartitionsByExpr(List partColumnNames, - List partColumnTypeInfos, byte[] expr, - String defaultPartitionName, List partitionNames) throws MetaException; + boolean filterPartitionsByExpr(List partColumns, + byte[] expr, String defaultPartitionName, List partitionNames) throws MetaException; /** * Determines the file metadata type from input format of the source table or partition. diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 697cc2e8c17..dd4ac96624a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -50,7 +50,6 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; @@ -82,8 +81,6 @@ import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hive.common.util.HiveStringUtils; import org.apache.thrift.TException; import org.slf4j.Logger; @@ -1046,17 +1043,10 @@ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, for (Partition part : parts) { result.add(Warehouse.makePartName(table.getPartitionKeys(), part.getValues())); } - List columnNames = new ArrayList(); - List typeInfos = new ArrayList(); - for (FieldSchema fs : table.getPartitionKeys()) { - columnNames.add(fs.getName()); - typeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(fs.getType())); - } if (defaultPartName == null || defaultPartName.isEmpty()) { defaultPartName = HiveConf.getVar(getConf(), HiveConf.ConfVars.DEFAULTPARTITIONNAME); } - return expressionProxy.filterPartitionsByExpr( - columnNames, typeInfos, expr, defaultPartName, result); + return expressionProxy.filterPartitionsByExpr(table.getPartitionKeys(), expr, defaultPartName, result); } @Override diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java b/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java index 8b12899d985..12773ac9e97 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java @@ -26,13 +26,13 @@ import org.antlr.runtime.ANTLRStringStream; import org.antlr.runtime.CharStream; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.ColumnType; import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; -import org.apache.hadoop.hive.serde.serdeConstants; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; @@ -432,8 +432,8 @@ private String getJdoFilterPushdownParam(Table table, int partColIndex, boolean isIntegralSupported = canPushDownIntegral && canJdoUseStringsWithIntegral(); String colType = table.getPartitionKeys().get(partColIndex).getType(); // Can only support partitions whose types are string, or maybe integers - if (!colType.equals(serdeConstants.STRING_TYPE_NAME) - && (!isIntegralSupported || !serdeConstants.IntegralTypes.contains(colType))) { + if (!colType.equals(ColumnType.STRING_TYPE_NAME) + && (!isIntegralSupported || !ColumnType.IntegralTypes.contains(colType))) { filterBuilder.setError("Filtering is supported only on partition keys of type " + "string" + (isIntegralSupported ? ", or integral types" : "")); return null; diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/MockPartitionExpressionForMetastore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/MockPartitionExpressionForMetastore.java index 032af69259c..12a862d6bdc 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/MockPartitionExpressionForMetastore.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/MockPartitionExpressionForMetastore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; @@ -35,8 +36,8 @@ public String convertExprToFilter(byte[] expr) throws MetaException { } @Override - public boolean filterPartitionsByExpr(List partColumnNames, - List partColumnTypeInfos, byte[] expr, String defaultPartitionName, + public boolean filterPartitionsByExpr(List partColumns, + byte[] expr, String defaultPartitionName, List partitionNames) throws MetaException { return false; } diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java index b28ea735935..d3a2bd9bd9b 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -52,8 +52,6 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.messaging.EventMessage; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -88,9 +86,8 @@ public String convertExprToFilter(byte[] expr) throws MetaException { } @Override - public boolean filterPartitionsByExpr(List partColumnNames, - List partColumnTypeInfos, byte[] expr, - String defaultPartitionName, List partitionNames) + public boolean filterPartitionsByExpr(List partColumns, + byte[] expr, String defaultPartitionName, List partitionNames) throws MetaException { return false; } @@ -280,8 +277,8 @@ public void testPartitionOps() throws MetaException, InvalidObjectException, NoS StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null); HashMap tableParams = new HashMap(); tableParams.put("EXTERNAL", "false"); - FieldSchema partitionKey1 = new FieldSchema("Country", serdeConstants.STRING_TYPE_NAME, ""); - FieldSchema partitionKey2 = new FieldSchema("State", serdeConstants.STRING_TYPE_NAME, ""); + FieldSchema partitionKey1 = new FieldSchema("Country", ColumnType.STRING_TYPE_NAME, ""); + FieldSchema partitionKey2 = new FieldSchema("State", ColumnType.STRING_TYPE_NAME, ""); Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, Arrays.asList(partitionKey1, partitionKey2), tableParams, null, null, "MANAGED_TABLE"); objectStore.createTable(tbl1); HashMap partitionParams = new HashMap(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java index 65551874bf2..3d8852eee9f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java @@ -18,8 +18,10 @@ package org.apache.hadoop.hive.ql.optimizer.ppr; +import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; +import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hive.metastore.FileFormatProxy; @@ -35,6 +37,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,9 +53,14 @@ public String convertExprToFilter(byte[] exprBytes) throws MetaException { } @Override - public boolean filterPartitionsByExpr(List partColumnNames, - List partColumnTypeInfos, byte[] exprBytes, - String defaultPartitionName, List partitionNames) throws MetaException { + public boolean filterPartitionsByExpr(List partColumns, + byte[] exprBytes, String defaultPartitionName, List partitionNames) throws MetaException { + List partColumnNames = new ArrayList<>(); + List partColumnTypeInfos = new ArrayList<>(); + for (FieldSchema fs : partColumns) { + partColumnNames.add(fs.getName()); + partColumnTypeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(fs.getType())); + } ExprNodeGenericFuncDesc expr = deserializeExpr(exprBytes); try { ExprNodeDescUtils.replaceEqualDefaultPartition(expr, defaultPartitionName); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java new file mode 100644 index 00000000000..a41b5ee4625 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.utils.StringUtils; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Constants and utility functions for column types. This is explicitly done as constants in the + * class rather than an enum in order to interoperate with Hive's old serdeConstants. All type + * names in this class match the type names in Hive's serdeConstants class. They must continue + * to do so. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public class ColumnType { + public static final String VOID_TYPE_NAME = "void"; + + public static final String BOOLEAN_TYPE_NAME = "boolean"; + + public static final String TINYINT_TYPE_NAME = "tinyint"; + + public static final String SMALLINT_TYPE_NAME = "smallint"; + + public static final String INT_TYPE_NAME = "int"; + + public static final String BIGINT_TYPE_NAME = "bigint"; + + public static final String FLOAT_TYPE_NAME = "float"; + + public static final String DOUBLE_TYPE_NAME = "double"; + + public static final String STRING_TYPE_NAME = "string"; + + public static final String CHAR_TYPE_NAME = "char"; + + public static final String VARCHAR_TYPE_NAME = "varchar"; + + public static final String DATE_TYPE_NAME = "date"; + + public static final String DATETIME_TYPE_NAME = "datetime"; + + public static final String TIMESTAMP_TYPE_NAME = "timestamp"; + + public static final String DECIMAL_TYPE_NAME = "decimal"; + + public static final String BINARY_TYPE_NAME = "binary"; + + public static final String INTERVAL_YEAR_MONTH_TYPE_NAME = "interval_year_month"; + + public static final String INTERVAL_DAY_TIME_TYPE_NAME = "interval_day_time"; + + public static final String TIMESTAMPTZ_TYPE_NAME = "timestamp with time zone"; + + public static final String LIST_TYPE_NAME = "array"; + + public static final String MAP_TYPE_NAME = "map"; + + public static final String STRUCT_TYPE_NAME = "struct"; + + public static final String UNION_TYPE_NAME = "uniontype"; + + public static final String LIST_COLUMNS = "columns"; + + public static final String LIST_COLUMN_TYPES = "columns.types"; + + public static final String COLUMN_NAME_DELIMITER = "column.name.delimiter"; + + public static final String SERIALIZATION_FORMAT = "serialization.format"; + + public static final Set PrimitiveTypes = StringUtils.asSet( + VOID_TYPE_NAME, + BOOLEAN_TYPE_NAME, + TINYINT_TYPE_NAME, + SMALLINT_TYPE_NAME, + INT_TYPE_NAME, + BIGINT_TYPE_NAME, + FLOAT_TYPE_NAME, + DOUBLE_TYPE_NAME, + STRING_TYPE_NAME, + VARCHAR_TYPE_NAME, + CHAR_TYPE_NAME, + DATE_TYPE_NAME, + DATETIME_TYPE_NAME, + TIMESTAMP_TYPE_NAME, + INTERVAL_YEAR_MONTH_TYPE_NAME, + INTERVAL_DAY_TIME_TYPE_NAME, + DECIMAL_TYPE_NAME, + BINARY_TYPE_NAME, + TIMESTAMPTZ_TYPE_NAME); + + public static final Set StringTypes = StringUtils.asSet( + STRING_TYPE_NAME, + VARCHAR_TYPE_NAME, + CHAR_TYPE_NAME + ); + + public static final Set NumericTypes = StringUtils.asSet( + TINYINT_TYPE_NAME, + SMALLINT_TYPE_NAME, + INT_TYPE_NAME, + BIGINT_TYPE_NAME, + FLOAT_TYPE_NAME, + DOUBLE_TYPE_NAME, + DECIMAL_TYPE_NAME + ); + + // This intentionally does not include interval types. + public static final Set DateTimeTypes = StringUtils.asSet( + DATE_TYPE_NAME, + DATETIME_TYPE_NAME, + TIMESTAMP_TYPE_NAME, + TIMESTAMPTZ_TYPE_NAME + ); + + // This map defines the progression of up casts in numeric types. + public static final Map NumericCastOrder = new HashMap<>(); + + static { + NumericCastOrder.put(TINYINT_TYPE_NAME, 1); + NumericCastOrder.put(SMALLINT_TYPE_NAME, 2); + NumericCastOrder.put(INT_TYPE_NAME, 3); + NumericCastOrder.put(BIGINT_TYPE_NAME, 4); + NumericCastOrder.put(DECIMAL_TYPE_NAME, 5); + NumericCastOrder.put(FLOAT_TYPE_NAME, 6); + NumericCastOrder.put(DOUBLE_TYPE_NAME, 7); + } + + private static final Map alternateTypeNames = new HashMap<>(); + + static { + alternateTypeNames.put("integer", INT_TYPE_NAME); + alternateTypeNames.put("numeric", DECIMAL_TYPE_NAME); + } + + public static final Set CollectionTypes = StringUtils.asSet( + LIST_TYPE_NAME, + MAP_TYPE_NAME); + + public static final Set IntegralTypes = StringUtils.asSet( + TINYINT_TYPE_NAME, + SMALLINT_TYPE_NAME, + INT_TYPE_NAME, + BIGINT_TYPE_NAME); + + public static final Set AllTypes = StringUtils.asSet( + VOID_TYPE_NAME, + BOOLEAN_TYPE_NAME, + TINYINT_TYPE_NAME, + SMALLINT_TYPE_NAME, + INT_TYPE_NAME, + BIGINT_TYPE_NAME, + FLOAT_TYPE_NAME, + DOUBLE_TYPE_NAME, + STRING_TYPE_NAME, + CHAR_TYPE_NAME, + VARCHAR_TYPE_NAME, + DATE_TYPE_NAME, + DATETIME_TYPE_NAME, + TIMESTAMP_TYPE_NAME, + DECIMAL_TYPE_NAME, + BINARY_TYPE_NAME, + INTERVAL_YEAR_MONTH_TYPE_NAME, + INTERVAL_DAY_TIME_TYPE_NAME, + TIMESTAMPTZ_TYPE_NAME, + LIST_TYPE_NAME, + MAP_TYPE_NAME, + STRUCT_TYPE_NAME, + UNION_TYPE_NAME, + LIST_COLUMNS, + LIST_COLUMN_TYPES, + COLUMN_NAME_DELIMITER + ); + + /** + * Given a type string return the type name. For example, passing in the type string + * varchar(256) will return varchar. + * @param typeString Type string + * @return type name, guaranteed to be in lower case + */ + public static String getTypeName(String typeString) { + if (typeString == null) return null; + String protoType = typeString.toLowerCase().split("\\W")[0]; + String realType = alternateTypeNames.get(protoType); + return realType == null ? protoType : realType; + } + + public static boolean areColTypesCompatible(String from, String to) { + if (from.equals(to)) return true; + + if (PrimitiveTypes.contains(from) && PrimitiveTypes.contains(to)) { + // They aren't the same, but we may be able to do a cast + + // If they are both types of strings, that should be fine + if (StringTypes.contains(from) && StringTypes.contains(to)) return true; + + // If both are numeric, make sure the new type is larger than the old. + if (NumericTypes.contains(from) && NumericTypes.contains(to)) { + return NumericCastOrder.get(from) < NumericCastOrder.get(to); + } + + // Allow string to double conversion + if (StringTypes.contains(from) && to.equals(DOUBLE_TYPE_NAME)) return true; + + // Void can go to anything + if (from.equals(VOID_TYPE_NAME)) return true; + + // Allow date to string casts. NOTE: I suspect this is the reverse of what we actually + // want, but it matches the code in o.a.h.h.serde2.typeinfo.TypeInfoUtils. I can't see how + // users would be altering date columns into string columns. The other I easily see since + // Hive did not originally support datetime types. Also, the comment in the Hive code + // says string to date, even though the code does the opposite. But for now I'm keeping + // this as is so the functionality matches. + if (DateTimeTypes.contains(from) && StringTypes.contains(to)) return true; + + // Allow numeric to string + if (NumericTypes.contains(from) && StringTypes.contains(to)) return true; + + } + return false; + } +} From 1e5e99e897df83bd2384f181797bbd9d4b810179 Mon Sep 17 00:00:00 2001 From: Alan Gates Date: Thu, 17 Aug 2017 16:32:41 -0700 Subject: [PATCH 2/2] Fixed broken compilation in test. --- .../src/test/org/apache/hadoop/hive/metastore/TestOldSchema.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestOldSchema.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestOldSchema.java index 8312b346a5e..8409d9bacc8 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestOldSchema.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestOldSchema.java @@ -62,9 +62,9 @@ public String convertExprToFilter(byte[] expr) throws MetaException { } @Override - public boolean filterPartitionsByExpr(List partColumnNames, - List partColumnTypeInfos, byte[] expr, String defaultPartitionName, - List partitionNames) throws MetaException { + public boolean filterPartitionsByExpr(List partColumns, byte[] expr, + String defaultPartitionName, + List partitionNames) throws MetaException { return false; }