commit 4efddeced35a554b03034892865ded7d7c2d8bb4 Author: Vihang Karajgaonkar Date: Thu May 3 10:25:16 2018 -0700 HIVE-19041 : Thrift deserialization of Partition objects should intern fields diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java index ff634081db8e6f0f9752c74fafd7f3ce81dbd312..8f00fbdd246aff755f1071fd2048626ff0a5b50d 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java @@ -491,7 +491,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, FieldSchema struct) switch (schemeField.id) { case 1: // NAME if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.name = iprot.readString(); + struct.name = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setNameIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -499,7 +499,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, FieldSchema struct) break; case 2: // TYPE if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.type = iprot.readString(); + struct.type = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setTypeIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -507,7 +507,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, FieldSchema struct) break; case 3: // COMMENT if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.comment = iprot.readString(); + struct.comment = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setCommentIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -585,15 +585,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, FieldSchema struct) TTupleProtocol iprot = (TTupleProtocol) prot; BitSet incoming = iprot.readBitSet(3); if (incoming.get(0)) { - struct.name = iprot.readString(); + struct.name = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setNameIsSet(true); } if (incoming.get(1)) { - struct.type = iprot.readString(); + struct.type = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setTypeIsSet(true); } if (incoming.get(2)) { - struct.comment = iprot.readString(); + struct.comment = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setCommentIsSet(true); } } diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java index c58e1cb7d93d2db13ee7e2d7ceddd62f1d8b81ff..51f809a0f8d483d48b69a5a0e75b1c141bc7a716 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java @@ -231,7 +231,7 @@ public Partition(Partition other) { this.privileges = new PrincipalPrivilegeSet(other.privileges); } if (other.isSetCatName()) { - this.catName = other.catName; + this.catName = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(other.catName); } } @@ -467,7 +467,7 @@ public String getCatName() { } public void setCatName(String catName) { - this.catName = catName; + this.catName = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(catName); } public void unsetCatName() { @@ -1029,7 +1029,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Partition struct) t break; case 2: // DB_NAME if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.dbName = iprot.readString(); + struct.dbName = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setDbNameIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -1037,7 +1037,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Partition struct) t break; case 3: // TABLE_NAME if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.tableName = iprot.readString(); + struct.tableName = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setTableNameIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -1099,7 +1099,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Partition struct) t break; case 9: // CAT_NAME if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.catName = iprot.readString(); + struct.catName = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setCatNameIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -1284,11 +1284,11 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Partition struct) th struct.setValuesIsSet(true); } if (incoming.get(1)) { - struct.dbName = iprot.readString(); + struct.dbName = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setDbNameIsSet(true); } if (incoming.get(2)) { - struct.tableName = iprot.readString(); + struct.tableName = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setTableNameIsSet(true); } if (incoming.get(3)) { @@ -1325,7 +1325,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Partition struct) th struct.setPrivilegesIsSet(true); } if (incoming.get(8)) { - struct.catName = iprot.readString(); + struct.catName = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setCatNameIsSet(true); } } diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java index 71957f79f27927410899c9c6ef83b20fb507371b..671c43efb3353a577b881d1a6327adaf662cb1f7 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java @@ -197,10 +197,10 @@ public SerDeInfo(SerDeInfo other) { this.description = other.description; } if (other.isSetSerializerClass()) { - this.serializerClass = other.serializerClass; + this.serializerClass = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(other.serializerClass); } if (other.isSetDeserializerClass()) { - this.deserializerClass = other.deserializerClass; + this.deserializerClass = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(other.deserializerClass); } if (other.isSetSerdeType()) { this.serdeType = other.serdeType; @@ -330,7 +330,7 @@ public String getSerializerClass() { } public void setSerializerClass(String serializerClass) { - this.serializerClass = serializerClass; + this.serializerClass = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(serializerClass); } public void unsetSerializerClass() { @@ -353,7 +353,7 @@ public String getDeserializerClass() { } public void setDeserializerClass(String deserializerClass) { - this.deserializerClass = deserializerClass; + this.deserializerClass = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(deserializerClass); } public void unsetDeserializerClass() { @@ -842,7 +842,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, SerDeInfo struct) t switch (schemeField.id) { case 1: // NAME if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.name = iprot.readString(); + struct.name = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setNameIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -850,7 +850,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, SerDeInfo struct) t break; case 2: // SERIALIZATION_LIB if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.serializationLib = iprot.readString(); + struct.serializationLib = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setSerializationLibIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -886,7 +886,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, SerDeInfo struct) t break; case 5: // SERIALIZER_CLASS if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.serializerClass = iprot.readString(); + struct.serializerClass = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setSerializerClassIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -894,7 +894,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, SerDeInfo struct) t break; case 6: // DESERIALIZER_CLASS if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.deserializerClass = iprot.readString(); + struct.deserializerClass = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setDeserializerClassIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -1047,11 +1047,11 @@ public void read(org.apache.thrift.protocol.TProtocol prot, SerDeInfo struct) th TTupleProtocol iprot = (TTupleProtocol) prot; BitSet incoming = iprot.readBitSet(7); if (incoming.get(0)) { - struct.name = iprot.readString(); + struct.name = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setNameIsSet(true); } if (incoming.get(1)) { - struct.serializationLib = iprot.readString(); + struct.serializationLib = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setSerializationLibIsSet(true); } if (incoming.get(2)) { @@ -1074,11 +1074,11 @@ public void read(org.apache.thrift.protocol.TProtocol prot, SerDeInfo struct) th struct.setDescriptionIsSet(true); } if (incoming.get(4)) { - struct.serializerClass = iprot.readString(); + struct.serializerClass = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setSerializerClassIsSet(true); } if (incoming.get(5)) { - struct.deserializerClass = iprot.readString(); + struct.deserializerClass = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setDeserializerClassIsSet(true); } if (incoming.get(6)) { diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java index 00e60417ffedab9666f6fafe67a09c2ee299fed5..6b79797a7b006e0e633ba861610d312bbb6885d4 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java @@ -1308,7 +1308,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, StorageDescriptor s break; case 2: // LOCATION if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.location = iprot.readString(); + struct.location = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setLocationIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -1316,7 +1316,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, StorageDescriptor s break; case 3: // INPUT_FORMAT if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.inputFormat = iprot.readString(); + struct.inputFormat = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setInputFormatIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -1324,7 +1324,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, StorageDescriptor s break; case 4: // OUTPUT_FORMAT if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.outputFormat = iprot.readString(); + struct.outputFormat = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setOutputFormatIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); @@ -1666,15 +1666,15 @@ public void read(org.apache.thrift.protocol.TProtocol prot, StorageDescriptor st struct.setColsIsSet(true); } if (incoming.get(1)) { - struct.location = iprot.readString(); + struct.location = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setLocationIsSet(true); } if (incoming.get(2)) { - struct.inputFormat = iprot.readString(); + struct.inputFormat = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setInputFormatIsSet(true); } if (incoming.get(3)) { - struct.outputFormat = iprot.readString(); + struct.outputFormat = org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot.readString()); struct.setOutputFormatIsSet(true); } if (incoming.get(4)) { diff --git a/standalone-metastore/src/main/resources/thrift-replacements.txt b/standalone-metastore/src/main/resources/thrift-replacements.txt index 01ee71a5b10f30d0514ad4cf41c2c767c1f86e5a..d7de0d3cf921a520f461ef032d16fa3ad47e189a 100644 --- a/standalone-metastore/src/main/resources/thrift-replacements.txt +++ b/standalone-metastore/src/main/resources/thrift-replacements.txt @@ -35,6 +35,9 @@ this\.inputFormat\ \=\ inputFormat;=this.inputFormat\ \=\ org.apache.hadoop.hive this\.outputFormat\ \=\ outputFormat;=this.outputFormat\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(outputFormat); this\.dbName\ \=\ dbName;=this.dbName\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(dbName); this\.tableName\ \=\ tableName;=this.tableName\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(tableName); +this\.catName\ \=\ catName;=this\.catName\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(catName); +this\.serializerClass\ \=\ serializerClass;=this\.serializerClass\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(serializerClass); +this\.deserializerClass\ \=\ deserializerClass;=this\.deserializerClass\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(deserializerClass); # Fix constructors and setters of List instance fields @@ -55,6 +58,9 @@ this\.inputFormat\ \=\ other\.inputFormat;=this.inputFormat\ \=\ org.apache.hado this\.outputFormat\ \=\ other\.outputFormat;=this.outputFormat\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(other.outputFormat); this\.dbName\ \=\ other\.dbName;=this.dbName\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(other.dbName); this\.tableName\ \=\ other\.tableName;=this.tableName\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(other.tableName); +this\.catName\ \=\ other\.catName;=this\.catName\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(other.catName); +this\.serializerClass\ \=\ other\.serializerClass;=this\.serializerClass\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(other.serializerClass); +this\.deserializerClass\ \=\ other\.deserializerClass;=this\.deserializerClass\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(other.deserializerClass); __this__parameters_copy_key\ \=\ other_element_key;=__this__parameters_copy_key\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(other_element_key); __this__parameters_copy_value\ \=\ other_element_value;=__this__parameters_copy_value\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(other_element_value); @@ -67,3 +73,25 @@ this\.parameters\.put\(key,\ val\);=this.parameters.put(org.apache.hadoop.hive.m # Fix the deserialization methods in Partitions.java: intern parameters after it's deserialized struct\.setParametersIsSet\(true\);=struct.parameters\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(struct.parameters);\ struct.setParametersIsSet(true); + +# Fix the StandardScheme read method which deserializes the fields into the thrift objects + +# PartitionStandardScheme - parameters are already interned above +struct\.dbName\ \=\ iprot\.readString\(\);=struct\.dbName\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); +struct\.tableName\ \=\ iprot\.readString\(\);=struct\.tableName\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); +struct\.catName\ \=\ iprot\.readString\(\);=struct\.catName\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); + +# StorageDescriptorStandardScheme - parameters are already interned above +struct\.location\ \=\ iprot\.readString\(\);=struct\.location\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); +struct\.inputFormat\ \=\ iprot\.readString\(\);=struct\.inputFormat\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); +struct\.outputFormat\ \=\ iprot\.readString\(\);=struct\.outputFormat\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); + +# SerDeInfoStandardScheme - parameters are already interned above +struct\.name\ \=\ iprot\.readString\(\);=struct\.name\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); +struct\.serializationLib\ \=\ iprot\.readString\(\);=struct\.serializationLib\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); +struct\.serializerClass\ \=\ iprot\.readString\(\);=struct\.serializerClass\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); +struct\.deserializerClass\ \=\ iprot\.readString\(\);=struct\.deserializerClass\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); + +# FieldSchemaStandardScheme - name field gets automatically handled above +struct\.type\ \=\ iprot\.readString\(\);=struct\.type\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); +struct\.comment\ \=\ iprot\.readString\(\);=struct\.comment\ \=\ org.apache.hadoop.hive.metastore.utils.StringUtils.intern(iprot\.readString\(\)); \ No newline at end of file