diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 21d1b46..95c09c0 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -286,24 +286,6 @@ public static void partitionTester(HiveMetaStoreClient client, HiveConf hiveConf } assertTrue("Partitions are not same", part.equals(part_get)); - // check null cols schemas for a partition - List vals6 = makeVals("2016-02-22 00:00:00", "16"); - Partition part6 = makePartitionObject(dbName, tblName, vals6, tbl, "/part5"); - part6.getSd().setCols(null); - LOG.info("Creating partition will null field schema"); - client.add_partition(part6); - LOG.info("Listing all partitions for table " + dbName + "." + tblName); - final List partitions = client.listPartitions(dbName, tblName, (short) -1); - boolean foundPart = false; - for (Partition p : partitions) { - if (p.getValues().equals(vals6)) { - assertNull(p.getSd().getCols()); - LOG.info("Found partition " + p + " having null field schema"); - foundPart = true; - } - } - assertTrue(foundPart); - String partName = "ds=" + FileUtils.escapePathName("2008-07-01 14:13:12") + "/hr=14"; String part2Name = "ds=" + FileUtils.escapePathName("2008-07-01 14:13:12") + "/hr=15"; String part3Name = "ds=" + FileUtils.escapePathName("2008-07-02 14:13:12") + "/hr=15"; @@ -337,7 +319,7 @@ public static void partitionTester(HiveMetaStoreClient client, HiveConf hiveConf partialVals.clear(); partialVals.add(""); partialNames = client.listPartitionNames(dbName, tblName, partialVals, (short) -1); - assertTrue("Should have returned 5 partition names", partialNames.size() == 5); + assertTrue("Should have returned 4 partition names", partialNames.size() == 4); assertTrue("Not all part names returned", partialNames.containsAll(partNames)); // Test partition listing with a partial spec - hr is specified but ds is not diff --git metastore/if/hive_metastore.thrift metastore/if/hive_metastore.thrift index baab31b..0cda553 100755 --- metastore/if/hive_metastore.thrift +++ metastore/if/hive_metastore.thrift @@ -275,7 +275,7 @@ struct SkewedInfo { // this object holds all the information about physical storage of the data belonging to a table struct StorageDescriptor { - 1: list cols, // required (refer to types defined above) + 1: list cols = [], // required (refer to types defined above) 2: string location, // defaults to //tablename 3: string inputFormat, // SequenceFileInputFormat (binary) or TextInputFormat` or custom format 4: string outputFormat, // SequenceFileOutputFormat (binary) or IgnoreKeyTextOutputFormat or custom format diff --git metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h index 6838133..32caaa4 100644 --- metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h +++ metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h @@ -1946,7 +1946,7 @@ inline std::ostream& operator<<(std::ostream& out, const SkewedInfo& obj) } typedef struct _StorageDescriptor__isset { - _StorageDescriptor__isset() : cols(false), location(false), inputFormat(false), outputFormat(false), compressed(false), numBuckets(false), serdeInfo(false), bucketCols(false), sortCols(false), parameters(false), skewedInfo(false), storedAsSubDirectories(false) {} + _StorageDescriptor__isset() : cols(true), location(false), inputFormat(false), outputFormat(false), compressed(false), numBuckets(false), serdeInfo(false), bucketCols(false), sortCols(false), parameters(false), skewedInfo(false), storedAsSubDirectories(false) {} bool cols :1; bool location :1; bool inputFormat :1; @@ -1967,6 +1967,7 @@ class StorageDescriptor { StorageDescriptor(const StorageDescriptor&); StorageDescriptor& operator=(const StorageDescriptor&); StorageDescriptor() : location(), inputFormat(), outputFormat(), compressed(0), numBuckets(0), storedAsSubDirectories(0) { + } virtual ~StorageDescriptor() throw(); diff --git metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java index 938f06b..1e8c1ae 100644 --- metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java +++ metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java @@ -204,6 +204,8 @@ public String getFieldName() { } public StorageDescriptor() { + this.cols = new ArrayList(); + } public StorageDescriptor( @@ -286,7 +288,8 @@ public StorageDescriptor deepCopy() { @Override public void clear() { - this.cols = null; + this.cols = new ArrayList(); + this.location = null; this.inputFormat = null; this.outputFormat = null; diff --git metastore/src/gen/thrift/gen-php/metastore/Types.php metastore/src/gen/thrift/gen-php/metastore/Types.php index b9af4ef..1c2526e 100644 --- metastore/src/gen/thrift/gen-php/metastore/Types.php +++ metastore/src/gen/thrift/gen-php/metastore/Types.php @@ -4056,7 +4056,8 @@ class StorageDescriptor { /** * @var \metastore\FieldSchema[] */ - public $cols = null; + public $cols = array( + ); /** * @var string */ diff --git metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py index 21c0390..8520200 100644 --- metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -2889,7 +2889,8 @@ class StorageDescriptor: thrift_spec = ( None, # 0 - (1, TType.LIST, 'cols', (TType.STRUCT,(FieldSchema, FieldSchema.thrift_spec)), None, ), # 1 + (1, TType.LIST, 'cols', (TType.STRUCT,(FieldSchema, FieldSchema.thrift_spec)), [ + ], ), # 1 (2, TType.STRING, 'location', None, None, ), # 2 (3, TType.STRING, 'inputFormat', None, None, ), # 3 (4, TType.STRING, 'outputFormat', None, None, ), # 4 @@ -2903,7 +2904,10 @@ class StorageDescriptor: (12, TType.BOOL, 'storedAsSubDirectories', None, None, ), # 12 ) - def __init__(self, cols=None, location=None, inputFormat=None, outputFormat=None, compressed=None, numBuckets=None, serdeInfo=None, bucketCols=None, sortCols=None, parameters=None, skewedInfo=None, storedAsSubDirectories=None,): + def __init__(self, cols=thrift_spec[1][4], location=None, inputFormat=None, outputFormat=None, compressed=None, numBuckets=None, serdeInfo=None, bucketCols=None, sortCols=None, parameters=None, skewedInfo=None, storedAsSubDirectories=None,): + if cols is self.thrift_spec[1][4]: + cols = [ + ] self.cols = cols self.location = location self.inputFormat = inputFormat diff --git metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb index c735932..b2c22bd 100644 --- metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -696,7 +696,8 @@ class StorageDescriptor STOREDASSUBDIRECTORIES = 12 FIELDS = { - COLS => {:type => ::Thrift::Types::LIST, :name => 'cols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}}, + COLS => {:type => ::Thrift::Types::LIST, :name => 'cols', :default => [ + ], :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}}, LOCATION => {:type => ::Thrift::Types::STRING, :name => 'location'}, INPUTFORMAT => {:type => ::Thrift::Types::STRING, :name => 'inputFormat'}, OUTPUTFORMAT => {:type => ::Thrift::Types::STRING, :name => 'outputFormat'}, diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 86565a4..562733a 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -155,6 +155,9 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, List parts = msdb.getPartitions(dbname, name, -1); for (Partition part : parts) { List oldCols = part.getSd().getCols(); + if (oldCols == null) { + oldCols = new ArrayList<>(); + } part.getSd().setCols(newt.getSd().getCols()); String oldPartName = Warehouse.makePartName(oldt.getPartitionKeys(), part.getValues()); updatePartColumnStatsForAlterColumns(msdb, part, oldPartName, part.getValues(), oldCols, part); @@ -776,7 +779,10 @@ private void updatePartColumnStats(RawStore msdb, String dbName, String tableNam return; } if (oldPartition.getSd() != null && newPart.getSd() != null) { - List oldCols = oldPartition.getSd().getCols(); + List oldCols = oldPartition.getSd().getCols(); + if (oldCols == null) { + oldCols = new ArrayList<>(); + } if (!MetaStoreUtils.areSameColumns(oldCols, newPart.getSd().getCols())) { updatePartColumnStatsForAlterColumns(msdb, oldPartition, oldPartName, partVals, oldCols, newPart); } @@ -800,7 +806,13 @@ private void alterTableUpdateTableColumnStats(RawStore msdb, try { List oldCols = oldTable.getSd().getCols(); + if (oldCols == null) { + oldCols = new ArrayList<>(); + } List newCols = newTable.getSd().getCols(); + if (newCols == null) { + newCols = new ArrayList<>(); + } List newStatsObjs = new ArrayList(); ColumnStatistics colStats = null; boolean updateColumnStats = true; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 9ea6ac4..97cb3e6 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -641,7 +641,9 @@ static boolean isCascadeNeededInAlterTable(Table oldTable, Table newTable) { } static boolean areSameColumns(List oldCols, List newCols) { - if (oldCols.size() != newCols.size()) { + if (oldCols == null || newCols == null) { + return oldCols == newCols; + } else if (oldCols.size() != newCols.size()) { return false; } else { for (int i = 0; i < oldCols.size(); i++) { @@ -682,7 +684,7 @@ static private boolean areColTypesCompatible(String oldType, String newType) { * validate column type * * if it is predefined, yes. otherwise no - * @param name + * @param type * @return */ static public String validateColumnType(String type) {