Index: conf/hive-default.xml
===================================================================
--- conf/hive-default.xml (revision 1061438)
+++ conf/hive-default.xml (working copy)
@@ -662,6 +662,12 @@
+ hive.metastore.cache.pinobjtypes
+ Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order
+ List of comma separated metastore object types that should be pinned in the cache
+
+
+
hive.optimize.reducededuplication
true
Remove extra map-reduce jobs if the data is already clustered by the same key which needs to be used again. This should always be set to true. Since it is a new feature, it has been made configurable.
Index: metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (revision 1061438)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (working copy)
@@ -20,6 +20,7 @@
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -108,6 +109,23 @@
NO_STATE, OPEN, COMMITED, ROLLBACK
}
+ /* Define object types to be pinned in L2 cache
+ * Note that the leading and trailing ',' is required
+ */
+ private static final Map PINCLASSMAP;
+ static {
+ Map map = new HashMap();
+ map.put(",table,", MTable.class);
+ map.put(",storagedescriptor,", MStorageDescriptor.class);
+ map.put(",serdeinfo,", MSerDeInfo.class);
+ map.put(",partition,", MPartition.class);
+ map.put(",database,", MDatabase.class);
+ map.put(",type,", MType.class);
+ map.put(",fieldschema,", MFieldSchema.class);
+ map.put(",order,", MOrder.class);
+ PINCLASSMAP = Collections.unmodifiableMap(map);
+ }
+
private boolean isInitialized = false;
private PersistenceManager pm = null;
private Configuration hiveConf;
@@ -219,14 +237,17 @@
pmf = JDOHelper.getPersistenceManagerFactory(prop);
DataStoreCache dsc = pmf.getDataStoreCache();
if (dsc != null) {
- dsc.pinAll(true, MTable.class);
- dsc.pinAll(true, MStorageDescriptor.class);
- dsc.pinAll(true, MSerDeInfo.class);
- dsc.pinAll(true, MPartition.class);
- dsc.pinAll(true, MDatabase.class);
- dsc.pinAll(true, MType.class);
- dsc.pinAll(true, MFieldSchema.class);
- dsc.pinAll(true, MOrder.class);
+ HiveConf conf = new HiveConf(ObjectStore.class);
+ String objTypes = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORE_CACHE_PINOBJTYPES);
+ if (objTypes != null && objTypes.length() > 0) {
+ objTypes = objTypes.toLowerCase();
+ objTypes = "," + objTypes + ",";
+ for (String k : PINCLASSMAP.keySet()) {
+ if (objTypes.contains(k)) {
+ dsc.pinAll(true, PINCLASSMAP.get(k));
+ }
+ }
+ }
}
}
return pmf;
Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
===================================================================
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1061438)
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy)
@@ -60,7 +60,7 @@
HiveConf.ConfVars.METASTORECONNECTURLKEY,
HiveConf.ConfVars.METASTOREATTEMPTS,
HiveConf.ConfVars.METASTOREINTERVAL,
- HiveConf.ConfVars.METASTOREFORCERELOADCONF,
+ HiveConf.ConfVars.METASTOREFORCERELOADCONF
};
/**
@@ -161,6 +161,7 @@
METASTORE_KERBEROS_KEYTAB_FILE("hive.metastore.kerberos.keytab.file", ""),
METASTORE_KERBEROS_PRINCIPAL("hive.metastore.kerberos.principal", ""),
METASTORE_USE_THRIFT_SASL("hive.metastore.sasl.enabled", false),
+ METASTORE_CACHE_PINOBJTYPES("hive.metastore.cache.pinobjtypes", "Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order"),
// Default parameters for creating tables
NEWTABLEDEFAULTPARA("hive.table.parameters.default",""),