Index: conf/hive-default.xml
===================================================================
--- conf/hive-default.xml (revision 1062921)
+++ conf/hive-default.xml (working copy)
@@ -662,6 +662,12 @@
+ hive.metastore.cache.pinobjtypes
+ Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order
+ List of comma separated metastore object types that should be pinned in the cache
+
+
+
hive.optimize.reducededuplication
true
Remove extra map-reduce jobs if the data is already clustered by the same key which needs to be used again. This should always be set to true. Since it is a new feature, it has been made configurable.
Index: metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (revision 1062921)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (working copy)
@@ -20,6 +20,7 @@
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -108,6 +109,20 @@
NO_STATE, OPEN, COMMITED, ROLLBACK
}
+ private static final Map PINCLASSMAP;
+ static {
+ Map map = new HashMap();
+ map.put("table", MTable.class);
+ map.put("storagedescriptor", MStorageDescriptor.class);
+ map.put("serdeinfo", MSerDeInfo.class);
+ map.put("partition", MPartition.class);
+ map.put("database", MDatabase.class);
+ map.put("type", MType.class);
+ map.put("fieldschema", MFieldSchema.class);
+ map.put("order", MOrder.class);
+ PINCLASSMAP = Collections.unmodifiableMap(map);
+ }
+
private boolean isInitialized = false;
private PersistenceManager pm = null;
private Configuration hiveConf;
@@ -219,14 +234,18 @@
pmf = JDOHelper.getPersistenceManagerFactory(prop);
DataStoreCache dsc = pmf.getDataStoreCache();
if (dsc != null) {
- dsc.pinAll(true, MTable.class);
- dsc.pinAll(true, MStorageDescriptor.class);
- dsc.pinAll(true, MSerDeInfo.class);
- dsc.pinAll(true, MPartition.class);
- dsc.pinAll(true, MDatabase.class);
- dsc.pinAll(true, MType.class);
- dsc.pinAll(true, MFieldSchema.class);
- dsc.pinAll(true, MOrder.class);
+ HiveConf conf = new HiveConf(ObjectStore.class);
+ String objTypes = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORE_CACHE_PINOBJTYPES);
+ if (objTypes != null && objTypes.length() > 0) {
+ objTypes = objTypes.toLowerCase();
+ String[] typeTokens = objTypes.split(",");
+ for (String type : typeTokens) {
+ type = type.trim();
+ if (PINCLASSMAP.containsKey(type)) {
+ dsc.pinAll(true, PINCLASSMAP.get(type));
+ }
+ }
+ }
}
}
return pmf;
Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
===================================================================
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1062921)
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy)
@@ -60,7 +60,7 @@
HiveConf.ConfVars.METASTORECONNECTURLKEY,
HiveConf.ConfVars.METASTOREATTEMPTS,
HiveConf.ConfVars.METASTOREINTERVAL,
- HiveConf.ConfVars.METASTOREFORCERELOADCONF,
+ HiveConf.ConfVars.METASTOREFORCERELOADCONF
};
/**
@@ -161,6 +161,7 @@
METASTORE_KERBEROS_KEYTAB_FILE("hive.metastore.kerberos.keytab.file", ""),
METASTORE_KERBEROS_PRINCIPAL("hive.metastore.kerberos.principal", ""),
METASTORE_USE_THRIFT_SASL("hive.metastore.sasl.enabled", false),
+ METASTORE_CACHE_PINOBJTYPES("hive.metastore.cache.pinobjtypes", "Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order"),
// Default parameters for creating tables
NEWTABLEDEFAULTPARA("hive.table.parameters.default",""),