Index: conf/hive-default.xml =================================================================== --- conf/hive-default.xml (revision 1061438) +++ conf/hive-default.xml (working copy) @@ -662,6 +662,12 @@ + hive.metastore.cache.pinobjtypes + Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order + List of comma separated metastore object types that should be pinned in the cache + + + hive.optimize.reducededuplication true Remove extra map-reduce jobs if the data is already clustered by the same key which needs to be used again. This should always be set to true. Since it is a new feature, it has been made configurable. Index: metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (revision 1061438) +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (working copy) @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -108,6 +109,23 @@ NO_STATE, OPEN, COMMITED, ROLLBACK } + /* Define object types to be pinned in L2 cache + * Note that the leading and trailing ',' is required + */ + private static final Map PINCLASSMAP; + static { + Map map = new HashMap(); + map.put(",table,", MTable.class); + map.put(",storagedescriptor,", MStorageDescriptor.class); + map.put(",serdeinfo,", MSerDeInfo.class); + map.put(",partition,", MPartition.class); + map.put(",database,", MDatabase.class); + map.put(",type,", MType.class); + map.put(",fieldschema,", MFieldSchema.class); + map.put(",order,", MOrder.class); + PINCLASSMAP = Collections.unmodifiableMap(map); + } + private boolean isInitialized = false; private PersistenceManager pm = null; private Configuration hiveConf; @@ -219,14 +237,17 @@ pmf = JDOHelper.getPersistenceManagerFactory(prop); DataStoreCache dsc = pmf.getDataStoreCache(); if (dsc != null) { - dsc.pinAll(true, MTable.class); - dsc.pinAll(true, MStorageDescriptor.class); - dsc.pinAll(true, MSerDeInfo.class); - dsc.pinAll(true, MPartition.class); - dsc.pinAll(true, MDatabase.class); - dsc.pinAll(true, MType.class); - dsc.pinAll(true, MFieldSchema.class); - dsc.pinAll(true, MOrder.class); + HiveConf conf = new HiveConf(ObjectStore.class); + String objTypes = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORE_CACHE_PINOBJTYPES); + if (objTypes != null && objTypes.length() > 0) { + objTypes = objTypes.toLowerCase(); + objTypes = "," + objTypes + ","; + for (String k : PINCLASSMAP.keySet()) { + if (objTypes.contains(k)) { + dsc.pinAll(true, PINCLASSMAP.get(k)); + } + } + } } } return pmf; Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1061438) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -60,7 +60,7 @@ HiveConf.ConfVars.METASTORECONNECTURLKEY, HiveConf.ConfVars.METASTOREATTEMPTS, HiveConf.ConfVars.METASTOREINTERVAL, - HiveConf.ConfVars.METASTOREFORCERELOADCONF, + HiveConf.ConfVars.METASTOREFORCERELOADCONF }; /** @@ -161,6 +161,7 @@ METASTORE_KERBEROS_KEYTAB_FILE("hive.metastore.kerberos.keytab.file", ""), METASTORE_KERBEROS_PRINCIPAL("hive.metastore.kerberos.principal", ""), METASTORE_USE_THRIFT_SASL("hive.metastore.sasl.enabled", false), + METASTORE_CACHE_PINOBJTYPES("hive.metastore.cache.pinobjtypes", "Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order"), // Default parameters for creating tables NEWTABLEDEFAULTPARA("hive.table.parameters.default",""),