diff --git common/src/java/org/apache/hive/common/util/HiveStringUtils.java common/src/java/org/apache/hive/common/util/HiveStringUtils.java index c21c937..2705f1e 100644 --- common/src/java/org/apache/hive/common/util/HiveStringUtils.java +++ common/src/java/org/apache/hive/common/util/HiveStringUtils.java @@ -33,9 +33,13 @@ import java.util.Date; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.HashMap; import java.util.Locale; import java.util.StringTokenizer; +import com.google.common.collect.Interner; +import com.google.common.collect.Interners; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceStability; @@ -57,10 +61,62 @@ public static final int SHUTDOWN_HOOK_PRIORITY = 0; private static final DecimalFormat decimalFormat; + + /** + * Maintain a String pool to reduce memory. + */ + private static final Interner STRING_INTERNER; + static { - NumberFormat numberFormat = NumberFormat.getNumberInstance(Locale.ENGLISH); - decimalFormat = (DecimalFormat) numberFormat; - decimalFormat.applyPattern("#.##"); + NumberFormat numberFormat = NumberFormat.getNumberInstance(Locale.ENGLISH); + decimalFormat = (DecimalFormat) numberFormat; + decimalFormat.applyPattern("#.##"); + + STRING_INTERNER = Interners.newWeakInterner(); + } + + /** + * Return the internalized string, or null if the given string is null. + * @param str The string to intern + * @return The identical string cached in the string pool. + */ + public static String intern(String str) { + if(str == null) { + return null; + } + return STRING_INTERNER.intern(str); + } + + /** + * Return an interned list with identical contents as the given list. + * @param list The list whose strings will be interned + * @return An identical list with its strings interned. + */ + public static List intern(List list) { + if(list == null) { + return null; + } + List newList = new ArrayList(list.size()); + for(String str : list) { + newList.add(intern(str)); + } + return newList; + } + + /** + * Return an interned map with identical contents as the given map. + * @param map The map whose strings will be interned + * @return An identical map with its strings interned. + */ + public static Map intern(Map map) { + if(map == null) { + return null; + } + Map newMap = new HashMap(map.size()); + for(Map.Entry entry : map.entrySet()) { + newMap.put(intern(entry.getKey()), intern(entry.getValue())); + } + return newMap; } /** diff --git metastore/pom.xml metastore/pom.xml index 61d4ddb..5c295f6 100644 --- metastore/pom.xml +++ metastore/pom.xml @@ -165,6 +165,39 @@ + + thriftif + + + + com.google.code.maven-replacer-plugin + replacer + 1.5.3 + + + process-thrift-sources + process-sources + + replace + + + + + ${basedir}/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ + + FieldSchema.java + Partition.java + SerDeInfo.java + StorageDescriptor.java + + ${basedir}/thrift-replacements.txt + true + false + + + + + diff --git metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java index a993810..c84fa29 100644 --- metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java +++ metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/FieldSchema.java @@ -135,9 +135,9 @@ public FieldSchema( String comment) { this(); - this.name = name; - this.type = type; - this.comment = comment; + this.name = org.apache.hive.common.util.HiveStringUtils.intern(name); + this.type = org.apache.hive.common.util.HiveStringUtils.intern(type); + this.comment = org.apache.hive.common.util.HiveStringUtils.intern(comment); } /** @@ -145,13 +145,13 @@ public FieldSchema( */ public FieldSchema(FieldSchema other) { if (other.isSetName()) { - this.name = other.name; + this.name = org.apache.hive.common.util.HiveStringUtils.intern(other.name); } if (other.isSetType()) { - this.type = other.type; + this.type = org.apache.hive.common.util.HiveStringUtils.intern(other.type); } if (other.isSetComment()) { - this.comment = other.comment; + this.comment = org.apache.hive.common.util.HiveStringUtils.intern(other.comment); } } @@ -171,7 +171,7 @@ public String getName() { } public void setName(String name) { - this.name = name; + this.name = org.apache.hive.common.util.HiveStringUtils.intern(name); } public void unsetName() { @@ -194,7 +194,7 @@ public String getType() { } public void setType(String type) { - this.type = type; + this.type = org.apache.hive.common.util.HiveStringUtils.intern(type); } public void unsetType() { @@ -217,7 +217,7 @@ public String getComment() { } public void setComment(String comment) { - this.comment = comment; + this.comment = org.apache.hive.common.util.HiveStringUtils.intern(comment); } public void unsetComment() { diff --git metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java index 312807e..242d54d 100644 --- metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java +++ metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Partition.java @@ -182,14 +182,14 @@ public Partition( { this(); this.values = values; - this.dbName = dbName; - this.tableName = tableName; + this.dbName = org.apache.hive.common.util.HiveStringUtils.intern(dbName); + this.tableName = org.apache.hive.common.util.HiveStringUtils.intern(tableName); this.createTime = createTime; setCreateTimeIsSet(true); this.lastAccessTime = lastAccessTime; setLastAccessTimeIsSet(true); this.sd = sd; - this.parameters = parameters; + this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters); } /** @@ -205,10 +205,10 @@ public Partition(Partition other) { this.values = __this__values; } if (other.isSetDbName()) { - this.dbName = other.dbName; + this.dbName = org.apache.hive.common.util.HiveStringUtils.intern(other.dbName); } if (other.isSetTableName()) { - this.tableName = other.tableName; + this.tableName = org.apache.hive.common.util.HiveStringUtils.intern(other.tableName); } this.createTime = other.createTime; this.lastAccessTime = other.lastAccessTime; @@ -222,9 +222,9 @@ public Partition(Partition other) { String other_element_key = other_element.getKey(); String other_element_value = other_element.getValue(); - String __this__parameters_copy_key = other_element_key; + String __this__parameters_copy_key = org.apache.hive.common.util.HiveStringUtils.intern(other_element_key); - String __this__parameters_copy_value = other_element_value; + String __this__parameters_copy_value = org.apache.hive.common.util.HiveStringUtils.intern(other_element_value); __this__parameters.put(__this__parameters_copy_key, __this__parameters_copy_value); } @@ -296,7 +296,7 @@ public String getDbName() { } public void setDbName(String dbName) { - this.dbName = dbName; + this.dbName = org.apache.hive.common.util.HiveStringUtils.intern(dbName); } public void unsetDbName() { @@ -319,7 +319,7 @@ public String getTableName() { } public void setTableName(String tableName) { - this.tableName = tableName; + this.tableName = org.apache.hive.common.util.HiveStringUtils.intern(tableName); } public void unsetTableName() { @@ -420,7 +420,7 @@ public void putToParameters(String key, String val) { } public void setParameters(Map parameters) { - this.parameters = parameters; + this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters); } public void unsetParameters() { diff --git metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java index 24d65bb..2466d8f 100644 --- metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java +++ metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/SerDeInfo.java @@ -137,9 +137,9 @@ public SerDeInfo( Map parameters) { this(); - this.name = name; - this.serializationLib = serializationLib; - this.parameters = parameters; + this.name = org.apache.hive.common.util.HiveStringUtils.intern(name); + this.serializationLib = org.apache.hive.common.util.HiveStringUtils.intern(serializationLib); + this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters); } /** @@ -147,10 +147,10 @@ public SerDeInfo( */ public SerDeInfo(SerDeInfo other) { if (other.isSetName()) { - this.name = other.name; + this.name = org.apache.hive.common.util.HiveStringUtils.intern(other.name); } if (other.isSetSerializationLib()) { - this.serializationLib = other.serializationLib; + this.serializationLib = org.apache.hive.common.util.HiveStringUtils.intern(other.serializationLib); } if (other.isSetParameters()) { Map __this__parameters = new HashMap(); @@ -159,9 +159,9 @@ public SerDeInfo(SerDeInfo other) { String other_element_key = other_element.getKey(); String other_element_value = other_element.getValue(); - String __this__parameters_copy_key = other_element_key; + String __this__parameters_copy_key = org.apache.hive.common.util.HiveStringUtils.intern(other_element_key); - String __this__parameters_copy_value = other_element_value; + String __this__parameters_copy_value = org.apache.hive.common.util.HiveStringUtils.intern(other_element_value); __this__parameters.put(__this__parameters_copy_key, __this__parameters_copy_value); } @@ -185,7 +185,7 @@ public String getName() { } public void setName(String name) { - this.name = name; + this.name = org.apache.hive.common.util.HiveStringUtils.intern(name); } public void unsetName() { @@ -208,7 +208,7 @@ public String getSerializationLib() { } public void setSerializationLib(String serializationLib) { - this.serializationLib = serializationLib; + this.serializationLib = org.apache.hive.common.util.HiveStringUtils.intern(serializationLib); } public void unsetSerializationLib() { @@ -242,7 +242,7 @@ public void putToParameters(String key, String val) { } public void setParameters(Map parameters) { - this.parameters = parameters; + this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters); } public void unsetParameters() { diff --git metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java index d0b9843..b91cc1c 100644 --- metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java +++ metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java @@ -216,17 +216,17 @@ public StorageDescriptor( { this(); this.cols = cols; - this.location = location; - this.inputFormat = inputFormat; - this.outputFormat = outputFormat; + this.location = org.apache.hive.common.util.HiveStringUtils.intern(location); + this.inputFormat = org.apache.hive.common.util.HiveStringUtils.intern(inputFormat); + this.outputFormat = org.apache.hive.common.util.HiveStringUtils.intern(outputFormat); this.compressed = compressed; setCompressedIsSet(true); this.numBuckets = numBuckets; setNumBucketsIsSet(true); this.serdeInfo = serdeInfo; - this.bucketCols = bucketCols; + this.bucketCols = org.apache.hive.common.util.HiveStringUtils.intern(bucketCols); this.sortCols = sortCols; - this.parameters = parameters; + this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters); } /** @@ -242,13 +242,13 @@ public StorageDescriptor(StorageDescriptor other) { this.cols = __this__cols; } if (other.isSetLocation()) { - this.location = other.location; + this.location = org.apache.hive.common.util.HiveStringUtils.intern(other.location); } if (other.isSetInputFormat()) { - this.inputFormat = other.inputFormat; + this.inputFormat = org.apache.hive.common.util.HiveStringUtils.intern(other.inputFormat); } if (other.isSetOutputFormat()) { - this.outputFormat = other.outputFormat; + this.outputFormat = org.apache.hive.common.util.HiveStringUtils.intern(other.outputFormat); } this.compressed = other.compressed; this.numBuckets = other.numBuckets; @@ -276,9 +276,9 @@ public StorageDescriptor(StorageDescriptor other) { String other_element_key = other_element.getKey(); String other_element_value = other_element.getValue(); - String __this__parameters_copy_key = other_element_key; + String __this__parameters_copy_key = org.apache.hive.common.util.HiveStringUtils.intern(other_element_key); - String __this__parameters_copy_value = other_element_value; + String __this__parameters_copy_value = org.apache.hive.common.util.HiveStringUtils.intern(other_element_value); __this__parameters.put(__this__parameters_copy_key, __this__parameters_copy_value); } @@ -356,7 +356,7 @@ public String getLocation() { } public void setLocation(String location) { - this.location = location; + this.location = org.apache.hive.common.util.HiveStringUtils.intern(location); } public void unsetLocation() { @@ -379,7 +379,7 @@ public String getInputFormat() { } public void setInputFormat(String inputFormat) { - this.inputFormat = inputFormat; + this.inputFormat = org.apache.hive.common.util.HiveStringUtils.intern(inputFormat); } public void unsetInputFormat() { @@ -402,7 +402,7 @@ public String getOutputFormat() { } public void setOutputFormat(String outputFormat) { - this.outputFormat = outputFormat; + this.outputFormat = org.apache.hive.common.util.HiveStringUtils.intern(outputFormat); } public void unsetOutputFormat() { @@ -507,7 +507,7 @@ public void addToBucketCols(String elem) { } public void setBucketCols(List bucketCols) { - this.bucketCols = bucketCols; + this.bucketCols = org.apache.hive.common.util.HiveStringUtils.intern(bucketCols); } public void unsetBucketCols() { @@ -579,7 +579,7 @@ public void putToParameters(String key, String val) { } public void setParameters(Map parameters) { - this.parameters = parameters; + this.parameters = org.apache.hive.common.util.HiveStringUtils.intern(parameters); } public void unsetParameters() { diff --git metastore/thrift-replacements.txt metastore/thrift-replacements.txt new file mode 100644 index 0000000..76ad55c --- /dev/null +++ metastore/thrift-replacements.txt @@ -0,0 +1,45 @@ +################################################################################################### +# # +# Used for the internalizing of String instance field assignments in the Thrift generated files # +# FieldSchema.java, Partition.java, SerDeInfo.java, and StorageDescriptor.java. # +# # +# Look in hive/metastore/pom.xml for the thriftif profile. # +# Usage: thriftif profile automatically refers to this file. # +# # +################################################################################################### + +# Fix constructors and setters of String instance fields + +this\.name\ \=\ name;=this.name\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(name); +this\.serializationLib\ \=\ serializationLib;=this.serializationLib\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(serializationLib); +this\.type\ \=\ type;=this.type\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(type); +this\.comment\ \=\ comment;=this.comment\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(comment); +this\.location\ \=\ location;=this.location\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(location); +this\.inputFormat\ \=\ inputFormat;=this.inputFormat\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(inputFormat); +this\.outputFormat\ \=\ outputFormat;=this.outputFormat\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(outputFormat); +this\.dbName\ \=\ dbName;=this.dbName\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(dbName); +this\.tableName\ \=\ tableName;=this.tableName\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(tableName); + +# Fix constructors and setters of List instance fields + +this\.bucketCols\ \=\ bucketCols;=this.bucketCols\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(bucketCols); + +# Fix constructors and setters of Map instance fields + +this\.parameters\ \=\ parameters;=this.parameters\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(parameters); + +# Fix copy constructors + +this\.name\ \=\ other\.name;=this.name\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.name); +this\.serializationLib\ \=\ other\.serializationLib;=this.serializationLib\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.serializationLib); +this\.type\ \=\ other\.type;=this.type\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.type); +this\.comment\ \=\ other\.comment;=this.comment\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.comment); +this\.location\ \=\ other\.location;=this.location\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.location); +this\.inputFormat\ \=\ other\.inputFormat;=this.inputFormat\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.inputFormat); +this\.outputFormat\ \=\ other\.outputFormat;=this.outputFormat\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.outputFormat); +this\.dbName\ \=\ other\.dbName;=this.dbName\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.dbName); +this\.tableName\ \=\ other\.tableName;=this.tableName\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other.tableName); + +__this__parameters_copy_key\ \=\ other_element_key;=__this__parameters_copy_key\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other_element_key); +__this__parameters_copy_value\ \=\ other_element_value;=__this__parameters_copy_value\ \=\ org.apache.hive.common.util.HiveStringUtils.intern(other_element_value); +__this_values\.add(other_element);=__this_values.add(org.apache.hive.common.util.HiveStringUtils.intern(other_element));