commit 8785bfa1ff874053f19aaefce87014229c27db90 Author: Vihang Karajgaonkar Date: Thu May 3 10:25:16 2018 -0700 HIVE-19041 : Thrift deserialization of Partition objects should intern fields diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java index ff634081db8e6f0f9752c74fafd7f3ce81dbd312..bb480edb0dad16e90a92476e4361e27983e75422 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java @@ -491,7 +491,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, FieldSchema struct) switch (schemeField.id) { case 1: // NAME if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.name = iprot.readString(); + struct.name = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setNameIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -585,7 +585,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, FieldSchema struct) TTupleProtocol iprot = (TTupleProtocol) prot; BitSet incoming = iprot.readBitSet(3); if (incoming.get(0)) { - struct.name = iprot.readString(); + struct.name = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setNameIsSet(true); } if (incoming.get(1)) { diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java index c58e1cb7d93d2db13ee7e2d7ceddd62f1d8b81ff..0ac81ed38c61f1c97c1cae88f42f974fa90ea2d7 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java @@ -1029,7 +1029,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Partition struct) t break; case 2: // DB_NAME if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.dbName = iprot.readString(); + struct.dbName = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setDbNameIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -1037,7 +1037,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Partition struct) t break; case 3: // TABLE_NAME if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.tableName = iprot.readString(); + struct.tableName = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setTableNameIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -1099,7 +1099,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Partition struct) t break; case 9: // CAT_NAME if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.catName = iprot.readString(); + struct.catName = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setCatNameIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -1284,11 +1284,11 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Partition struct) th struct.setValuesIsSet(true); } if (incoming.get(1)) { - struct.dbName = iprot.readString(); + struct.dbName = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setDbNameIsSet(true); } if (incoming.get(2)) { - struct.tableName = iprot.readString(); + struct.tableName = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setTableNameIsSet(true); } if (incoming.get(3)) { @@ -1325,7 +1325,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Partition struct) th struct.setPrivilegesIsSet(true); } if (incoming.get(8)) { - struct.catName = iprot.readString(); + struct.catName = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setCatNameIsSet(true); } } diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java index 71957f79f27927410899c9c6ef83b20fb507371b..4b696fd27a5c10854339bb6d73877b93e6086499 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java @@ -842,7 +842,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, SerDeInfo struct) t switch (schemeField.id) { case 1: // NAME if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.name = iprot.readString(); + struct.name = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setNameIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -850,7 +850,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, SerDeInfo struct) t break; case 2: // SERIALIZATION_LIB if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.serializationLib = iprot.readString(); + struct.serializationLib = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setSerializationLibIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -886,7 +886,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, SerDeInfo struct) t break; case 5: // SERIALIZER_CLASS if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.serializerClass = iprot.readString(); + struct.serializerClass = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setSerializerClassIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -894,7 +894,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, SerDeInfo struct) t break; case 6: // DESERIALIZER_CLASS if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.deserializerClass = iprot.readString(); + struct.deserializerClass = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setDeserializerClassIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -1047,11 +1047,11 @@ public void read(org.apache.thrift.protocol.TProtocol prot, SerDeInfo struct) th TTupleProtocol iprot = (TTupleProtocol) prot; BitSet incoming = iprot.readBitSet(7); if (incoming.get(0)) { - struct.name = iprot.readString(); + struct.name = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setNameIsSet(true); } if (incoming.get(1)) { - struct.serializationLib = iprot.readString(); + struct.serializationLib = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setSerializationLibIsSet(true); } if (incoming.get(2)) { @@ -1074,11 +1074,11 @@ public void read(org.apache.thrift.protocol.TProtocol prot, SerDeInfo struct) th struct.setDescriptionIsSet(true); } if (incoming.get(4)) { - struct.serializerClass = iprot.readString(); + struct.serializerClass = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setSerializerClassIsSet(true); } if (incoming.get(5)) { - struct.deserializerClass = iprot.readString(); + struct.deserializerClass = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setDeserializerClassIsSet(true); } if (incoming.get(6)) { diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java index 00e60417ffedab9666f6fafe67a09c2ee299fed5..6b79797a7b006e0e633ba861610d312bbb6885d4 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java @@ -1308,7 +1308,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, StorageDescriptor s break; case 2: // LOCATION if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.location = iprot.readString(); + struct.location = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setLocationIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -1316,7 +1316,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, StorageDescriptor s break; case 3: // INPUT_FORMAT if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.inputFormat = iprot.readString(); + struct.inputFormat = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setInputFormatIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -1324,7 +1324,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, StorageDescriptor s break; case 4: // OUTPUT_FORMAT if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.outputFormat = iprot.readString(); + struct.outputFormat = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setOutputFormatIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -1666,15 +1666,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, StorageDescriptor st struct.setColsIsSet(true); } if (incoming.get(1)) { - struct.location = iprot.readString(); + struct.location = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setLocationIsSet(true); } if (incoming.get(2)) { - struct.inputFormat = iprot.readString(); + struct.inputFormat = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setInputFormatIsSet(true); } if (incoming.get(3)) { - struct.outputFormat = iprot.readString(); + struct.outputFormat = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setOutputFormatIsSet(true); } if (incoming.get(4)) { diff --git a/standalone-metastore/src/main/resources/thrift-replacements.txt b/standalone-metastore/src/main/resources/thrift-replacements.txt index 01ee71a5b10f30d0514ad4cf41c2c767c1f86e5a..e23e49fd546cdb3e4c7628d8884021f7d71db620 100644 --- a/standalone-metastore/src/main/resources/thrift-replacements.txt +++ b/standalone-metastore/src/main/resources/thrift-replacements.txt @@ -67,3 +67,21 @@ this\.parameters\.put\(key,\ val\);=this.parameters.put(org.apache.hadoop.hive.m # Fix the deserialization methods in Partitions.java: intern parameters after it's deserialized struct\.setParametersIsSet\(true\);=struct.parameters\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(struct.parameters);\ struct.setParametersIsSet(true); + +# Fix the StandardScheme read method which deserializes the fields into the thrift objects + +# PartitionStandardScheme - parameters are already interned above +struct\.dbName\ \=\ iprot\.readString\(\);=struct\.dbName\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); +struct\.tableName\ \=\ iprot\.readString\(\);=struct\.tableName\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); +struct\.catName\ \=\ iprot\.readString\(\);=struct\.catName\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); + +# StorageDescriptorStandardScheme - parameters are already interned above +struct\.location\ \=\ iprot\.readString\(\);=struct\.location\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); +struct\.inputFormat\ \=\ iprot\.readString\(\);=struct\.inputFormat\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); +struct\.outputFormat\ \=\ iprot\.readString\(\);=struct\.outputFormat\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); + +# SerDeInfoStandardScheme - parameters are already interned above +struct\.name\ \=\ iprot\.readString\(\);=struct\.name\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); +struct\.serializationLib\ \=\ iprot\.readString\(\);=struct\.serializationLib\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); +struct\.serializerClass\ \=\ iprot\.readString\(\);=struct\.serializerClass\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); +struct\.deserializerClass\ \=\ iprot\.readString\(\);=struct\.deserializerClass\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); \ No newline at end of file