Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
===================================================================
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1182263)
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy)
@@ -86,6 +86,12 @@
HiveConf.ConfVars.METASTORE_IDENTIFIER_FACTORY,
HiveConf.ConfVars.METASTORE_PLUGIN_REGISTRY_BUNDLE_CHECK,
HiveConf.ConfVars.METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS,
+ HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX,
+ HiveConf.ConfVars.METASTORE_EVENT_LISTENERS,
+ HiveConf.ConfVars.METASTORE_EVENT_CLEAN_FREQ,
+ HiveConf.ConfVars.METASTORE_EVENT_EXPIRY_DURATION,
+ HiveConf.ConfVars.METASTORE_LOCAL,
+ HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
};
/**
@@ -110,7 +116,7 @@
// QL execution stuff
SCRIPTWRAPPER("hive.exec.script.wrapper", null),
PLAN("hive.exec.plan", null),
- SCRATCHDIR("hive.exec.scratchdir", "/tmp/" + System.getProperty("user.name") + "/hive"),
+ SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive-" + System.getProperty("user.name")),
SUBMITVIACHILD("hive.exec.submitviachild", false),
SCRIPTERRORLIMIT("hive.exec.script.maxerrsize", 100000),
ALLOWPARTIALCONSUMP("hive.exec.script.allow.partial.consumption", false),
@@ -162,14 +168,14 @@
HADOOPMAPREDINPUTDIR("mapred.input.dir", null),
HADOOPMAPREDINPUTDIRRECURSIVE("mapred.input.dir.recursive", false),
HADOOPJT("mapred.job.tracker", "local"),
- HADOOPNUMREDUCERS("mapred.reduce.tasks", 1),
+ HADOOPNUMREDUCERS("mapred.reduce.tasks", -1),
HADOOPJOBNAME("mapred.job.name", null),
HADOOPSPECULATIVEEXECREDUCERS("mapred.reduce.tasks.speculative.execution", false),
// Metastore stuff. Be sure to update HiveConf.metaVars when you add
// something here!
METASTOREDIRECTORY("hive.metastore.metadb.dir", ""),
- METASTOREWAREHOUSE("hive.metastore.warehouse.dir", ""),
+ METASTOREWAREHOUSE("hive.metastore.warehouse.dir", "/user/hive/warehouse"),
METASTOREURIS("hive.metastore.uris", ""),
// Number of times to retry a connection to a Thrift metastore server
METASTORETHRIFTRETRIES("hive.metastore.connect.retries", 5),
@@ -177,12 +183,13 @@
METASTORE_CLIENT_CONNECT_RETRY_DELAY("hive.metastore.client.connect.retry.delay", 1),
// Socket timeout for the client connection (in seconds)
METASTORE_CLIENT_SOCKET_TIMEOUT("hive.metastore.client.socket.timeout", 20),
- METASTOREPWD("javax.jdo.option.ConnectionPassword", ""),
+ METASTOREPWD("javax.jdo.option.ConnectionPassword", "mine"),
// Class name of JDO connection url hook
METASTORECONNECTURLHOOK("hive.metastore.ds.connection.url.hook", ""),
- METASTOREMULTITHREADED("javax.jdo.option.Multithreaded", "true"),
+ METASTOREMULTITHREADED("javax.jdo.option.Multithreaded", true),
// Name of the connection url in the configuration
- METASTORECONNECTURLKEY("javax.jdo.option.ConnectionURL", ""),
+ METASTORECONNECTURLKEY("javax.jdo.option.ConnectionURL",
+ "jdbc:derby:;databaseName=metastore_db;create=true"),
// Number of attempts to retry connecting after there is a JDO datastore err
METASTOREATTEMPTS("hive.metastore.ds.retry.attempts", 1),
// Number of miliseconds to wait between attepting
@@ -193,7 +200,7 @@
// testing only.
METASTOREFORCERELOADCONF("hive.metastore.force.reload.conf", false),
METASTORESERVERMINTHREADS("hive.metastore.server.min.threads", 200),
- METASTORESERVERMAXTHREADS("hive.metastore.server.max.threads", Integer.MAX_VALUE),
+ METASTORESERVERMAXTHREADS("hive.metastore.server.max.threads", 100000),
METASTORE_TCP_KEEP_ALIVE("hive.metastore.server.tcp.keepalive", true),
// Intermediate dir suffixes used for archiving. Not important what they
// are, as long as collisions are avoided
@@ -204,7 +211,8 @@
METASTORE_INT_EXTRACTED("hive.metastore.archive.intermediate.extracted",
"_INTERMEDIATE_EXTRACTED"),
METASTORE_KERBEROS_KEYTAB_FILE("hive.metastore.kerberos.keytab.file", ""),
- METASTORE_KERBEROS_PRINCIPAL("hive.metastore.kerberos.principal", ""),
+ METASTORE_KERBEROS_PRINCIPAL("hive.metastore.kerberos.principal",
+ "hive-metastore/_HOST@EXAMPLE.COM"),
METASTORE_USE_THRIFT_SASL("hive.metastore.sasl.enabled", false),
METASTORE_CACHE_PINOBJTYPES("hive.metastore.cache.pinobjtypes", "Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order"),
METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "DBCP"),
@@ -226,7 +234,17 @@
METASTORE_EVENT_CLEAN_FREQ("hive.metastore.event.clean.freq",0L),
METASTORE_EVENT_EXPIRY_DURATION("hive.metastore.event.expiry.duration",0L),
// Default parameters for creating tables
- NEWTABLEDEFAULTPARA("hive.table.parameters.default",""),
+ NEWTABLEDEFAULTPARA("hive.table.parameters.default", ""),
+ METASTORE_LOCAL("hive.metastore.local", true),
+ METASTORE_RAW_STORE_IMPL("hive.metastore.rawstore.impl",
+ "org.apache.hadoop.hive.metastore.ObjectStore"),
+ METASTORE_CONNECTION_DRIVER("javax.jdo.option.ConnectionDriverName",
+ "org.apache.derby.jdbc.EmbeddedDriver"),
+ METASTORE_MANAGER_FACTORY_CLASS("javax.jdo.PersistenceManagerFactoryClass",
+ "org.datanucleus.jdo.JDOPersistenceManagerFactory"),
+ METASTORE_DETACH_ALL_ON_COMMIT("javax.jdo.option.DetachAllOnCommit", true),
+ METASTORE_NON_TRANSACTIONAL_READ("javax.jdo.option.NonTransactionalRead", true),
+ METASTORE_CONNECTION_USER_NAME("javax.jdo.option.ConnectionUserName", "APP"),
// CLI
CLIIGNOREERRORS("hive.cli.errors.ignore", false),
@@ -334,7 +352,7 @@
HIVESKEWJOIN("hive.optimize.skewjoin", false),
HIVECONVERTJOIN("hive.auto.convert.join", false),
- HIVESKEWJOINKEY("hive.skewjoin.key", 1000000),
+ HIVESKEWJOINKEY("hive.skewjoin.key", 100000),
HIVESKEWJOINMAPJOINNUMMAPTASK("hive.skewjoin.mapjoin.map.tasks", 10000),
HIVESKEWJOINMAPJOINMINSPLIT("hive.skewjoin.mapjoin.min.split", 33554432L), //32M
MAPREDMAXSPLITSIZE("mapred.max.split.size", 256000000L),
@@ -424,7 +442,7 @@
HIVE_LOCK_MAPRED_ONLY("hive.lock.mapred.only.operation", false),
HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", ""),
- HIVE_ZOOKEEPER_CLIENT_PORT("hive.zookeeper.client.port", ""),
+ HIVE_ZOOKEEPER_CLIENT_PORT("hive.zookeeper.client.port", "2181"),
HIVE_ZOOKEEPER_SESSION_TIMEOUT("hive.zookeeper.session.timeout", 600*1000),
HIVE_ZOOKEEPER_NAMESPACE("hive.zookeeper.namespace", "hive_zookeeper_namespace"),
HIVE_ZOOKEEPER_CLEAN_EXTRA_NODES("hive.zookeeper.clean.extra.nodes", false),
@@ -450,13 +468,17 @@
SEMANTIC_ANALYZER_HOOK("hive.semantic.analyzer.hook",null),
HIVE_AUTHORIZATION_ENABLED("hive.security.authorization.enabled", false),
- HIVE_AUTHORIZATION_MANAGER("hive.security.authorization.manager", null),
- HIVE_AUTHENTICATOR_MANAGER("hive.security.authenticator.manager", null),
+ HIVE_AUTHORIZATION_MANAGER("hive.security.authorization.manager",
+ "org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider"),
+ HIVE_AUTHENTICATOR_MANAGER("hive.security.authenticator.manager",
+ "org.apache.hadoop.hive.ql.security.HadoopDefaultAuthenticator"),
+ HIVE_AUTHORIZATION_TABLE_USER_GRANTS("hive.security.authorization.createtable.user.grants", ""),
+ HIVE_AUTHORIZATION_TABLE_GROUP_GRANTS("hive.security.authorization.createtable.group.grants",
+ ""),
+ HIVE_AUTHORIZATION_TABLE_ROLE_GRANTS("hive.security.authorization.createtable.role.grants", ""),
+ HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS("hive.security.authorization.createtable.owner.grants",
+ ""),
- HIVE_AUTHORIZATION_TABLE_USER_GRANTS("hive.security.authorization.createtable.user.grants", null),
- HIVE_AUTHORIZATION_TABLE_GROUP_GRANTS("hive.security.authorization.createtable.group.grants", null),
- HIVE_AUTHORIZATION_TABLE_ROLE_GRANTS("hive.security.authorization.createtable.role.grants", null),
- HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS("hive.security.authorization.createtable.owner.grants", null),
// Print column names in output
HIVE_CLI_PRINT_HEADER("hive.cli.print.header", false),
@@ -471,6 +493,7 @@
HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS("hive.mapper.cannot.span.multiple.partitions", false),
HIVE_REWORK_MAPREDWORK("hive.rework.mapredwork", false),
HIVE_CONCATENATE_CHECK_INDEX ("hive.exec.concatenate.check.index", true),
+ HIVE_IO_EXCEPTION_HANDLERS("hive.io.exception.handlers", ""),
//prefix used to auto generated column aliases
HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL("hive.autogen.columnalias.prefix.label", "_c"),
@@ -480,6 +503,7 @@
// The class responsible for logging client side performance metrics
// Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger
HIVE_PERF_LOGGER("hive.exec.perf.logger", "org.apache.hadoop.hive.ql.log.PerfLogger"),
+ HIVE_FS_HAR_IMPL("fs.har.impl", "org.apache.hadoop.hive.shims.HiveHarFileSystem"),
// Whether to delete the scratchdir while startup
HIVE_START_CLEANUP_SCRATCHDIR("hive.start.cleanup.scratchdir", false),
;
Index: conf/hive-default.xml
===================================================================
--- conf/hive-default.xml (revision 1182263)
+++ conf/hive-default.xml (working copy)
@@ -379,13 +379,6 @@
- hive.optimize.pruner
- true
- Whether to enable the new partition pruner which depends on predicate pushdown. If this is disabled,
- the old partition pruner which is based on AST will be enabled.
-
-
-
hive.optimize.groupby
true
Whether to enable the bucketed group by from bucketed partitions/tables.
@@ -604,26 +597,7 @@
When the average output file size of a job is less than this number, Hive will start an additional map-reduce job to merge the output files into bigger files. This is only done for map-only jobs if hive.merge.mapfiles is true, and for map-reduce jobs if hive.merge.mapredfiles is true.
-
- hive.mapjoin.hashtable.initialCapacity
- 100000
- In the process of Mapjoin, the key/value will be hold in the hashtable. This value means the initialCapacity of the in-memory hashtable
-
-
-
- hive.mapjoin.hashtable.loadfactor
- 0.75
- In the process of Mapjoin, the key/value will be hold in the hashtable. This value means the load factor for the in-memory hashtable
-
-
-
- hive.mapjoin.smalltable.filesize
- 25000000
- The threshold for the input file size of the small tables; if the file size is smaller than this threshold, it will try to convert the common join into map join
-
-
-
hive.mapjoin.localtask.max.memory.usage
0.90
This number means how much memory the local task can take to hold the key/value into in-memory hash table; If the local task's memory usage is more than this number, the local task will be abort by themself. It means the data of small table is too large to be hold in the memory.
@@ -853,12 +827,6 @@
- hive.stats.jdbc.atomic
- false
- If this is set to true then the metastore stats will be updated only if all types of stats (# of rows, # of files, # of bytes etc.) are available. Otherwise metastore stats are updated in a best effort fashion with whatever are available.
-
-
-
hive.stats.retries.max
0
Maximum number of retries when stats publisher/aggregator got an exception updating intermediate database. Default is no tries on failures.
@@ -877,12 +845,6 @@
- hive.concurrency.manager
- org.apache.hadoop.hive.ql.lockmgr.ZooKeeperLockMgr
- The concurrency manager for hive.
-
-
-
hive.lock.numretries
100
The number of times you want to try to get all the locks
Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1182263)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy)
@@ -238,7 +238,7 @@
}
private boolean init() throws MetaException {
- rawStoreClassName = hiveConf.get("hive.metastore.rawstore.impl");
+ rawStoreClassName = hiveConf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL);
checkForDefaultDb = hiveConf.getBoolean(
"hive.metastore.checkForDefaultDb", true);
String alterHandlerName = hiveConf.get("hive.metastore.alter.impl",
Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (revision 1182263)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (working copy)
@@ -102,7 +102,7 @@
}
this.conf = conf;
- localMetaStore = conf.getBoolean("hive.metastore.local", false);
+ localMetaStore = conf.getBoolVar(HiveConf.ConfVars.METASTORE_LOCAL);
if (localMetaStore) {
// instantiate the metastore server handler directly instead of connecting
// through the network
Index: metastore/src/test/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java
===================================================================
--- metastore/src/test/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java (revision 1182263)
+++ metastore/src/test/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java (working copy)
@@ -41,7 +41,7 @@
Thread t = new Thread(new RunMS());
t.setDaemon(true);
t.start();
- hiveConf.set("hive.metastore.local", "false");
+ hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_LOCAL, false);
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:29111");
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTRETRIES, 3);
Thread.sleep(30000);
Index: metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java
===================================================================
--- metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java (revision 1182263)
+++ metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java (working copy)
@@ -39,7 +39,7 @@
public void setup() throws Exception {
System.setProperty(HiveConf.ConfVars.METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS.varname,
"true");
- conf.set("hive.metastore.local", "false");
+ conf.setBoolVar(HiveConf.ConfVars.METASTORE_LOCAL, false);
conf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
conf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTRETRIES, 3);
conf.setIntVar(ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, 60);
Index: metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
===================================================================
--- metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java (revision 1182263)
+++ metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java (working copy)
@@ -78,7 +78,7 @@
t.start();
Thread.sleep(40000);
hiveConf = new HiveConf(this.getClass());
- hiveConf.set("hive.metastore.local", "false");
+ hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_LOCAL, false);
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + msPort);
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTRETRIES, 3);
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
Index: metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
===================================================================
--- metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java (revision 1182263)
+++ metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java (working copy)
@@ -64,7 +64,7 @@
Thread.sleep(5000);
// hive.metastore.local should be defined in HiveConf
- hiveConf.set("hive.metastore.local", "false");
+ hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_LOCAL, false);
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + METASTORE_PORT);
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTRETRIES, 3);
hiveConf.setIntVar(ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, 60);
Index: shims/src/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java
===================================================================
--- shims/src/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java (revision 1182263)
+++ shims/src/test/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java (working copy)
@@ -117,7 +117,7 @@
System.setProperty(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, new Path(
System.getProperty("test.build.data", "/tmp")).toString());
conf = new HiveConf(TestHadoop20SAuthBridge.class);
- conf.setBoolean("hive.metastore.local", false);
+ conf.setBoolVar(HiveConf.ConfVars.METASTORE_LOCAL,false);
MetaStoreUtils.startMetaStore(port, new MyHadoopThriftAuthBridge20S());
}