Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1198626) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -87,6 +87,11 @@ HiveConf.ConfVars.METASTORE_IDENTIFIER_FACTORY, HiveConf.ConfVars.METASTORE_PLUGIN_REGISTRY_BUNDLE_CHECK, HiveConf.ConfVars.METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS, + HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX, + HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, + HiveConf.ConfVars.METASTORE_EVENT_CLEAN_FREQ, + HiveConf.ConfVars.METASTORE_EVENT_EXPIRY_DURATION, + HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, }; /** @@ -111,7 +116,7 @@ // QL execution stuff SCRIPTWRAPPER("hive.exec.script.wrapper", null), PLAN("hive.exec.plan", null), - SCRATCHDIR("hive.exec.scratchdir", "/tmp/" + System.getProperty("user.name") + "/hive"), + SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive-" + System.getProperty("user.name")), SUBMITVIACHILD("hive.exec.submitviachild", false), SCRIPTERRORLIMIT("hive.exec.script.maxerrsize", 100000), ALLOWPARTIALCONSUMP("hive.exec.script.allow.partial.consumption", false), @@ -163,14 +168,14 @@ HADOOPMAPREDINPUTDIR("mapred.input.dir", null), HADOOPMAPREDINPUTDIRRECURSIVE("mapred.input.dir.recursive", false), HADOOPJT("mapred.job.tracker", "local"), - HADOOPNUMREDUCERS("mapred.reduce.tasks", 1), + HADOOPNUMREDUCERS("mapred.reduce.tasks", -1), HADOOPJOBNAME("mapred.job.name", null), HADOOPSPECULATIVEEXECREDUCERS("mapred.reduce.tasks.speculative.execution", false), // Metastore stuff. Be sure to update HiveConf.metaVars when you add // something here! METASTOREDIRECTORY("hive.metastore.metadb.dir", ""), - METASTOREWAREHOUSE("hive.metastore.warehouse.dir", ""), + METASTOREWAREHOUSE("hive.metastore.warehouse.dir", "/user/hive/warehouse"), METASTOREURIS("hive.metastore.uris", ""), // Number of times to retry a connection to a Thrift metastore server METASTORETHRIFTRETRIES("hive.metastore.connect.retries", 5), @@ -178,12 +183,13 @@ METASTORE_CLIENT_CONNECT_RETRY_DELAY("hive.metastore.client.connect.retry.delay", 1), // Socket timeout for the client connection (in seconds) METASTORE_CLIENT_SOCKET_TIMEOUT("hive.metastore.client.socket.timeout", 20), - METASTOREPWD("javax.jdo.option.ConnectionPassword", ""), + METASTOREPWD("javax.jdo.option.ConnectionPassword", "mine"), // Class name of JDO connection url hook METASTORECONNECTURLHOOK("hive.metastore.ds.connection.url.hook", ""), - METASTOREMULTITHREADED("javax.jdo.option.Multithreaded", "true"), + METASTOREMULTITHREADED("javax.jdo.option.Multithreaded", true), // Name of the connection url in the configuration - METASTORECONNECTURLKEY("javax.jdo.option.ConnectionURL", ""), + METASTORECONNECTURLKEY("javax.jdo.option.ConnectionURL", + "jdbc:derby:;databaseName=metastore_db;create=true"), // Number of attempts to retry connecting after there is a JDO datastore err METASTOREATTEMPTS("hive.metastore.ds.retry.attempts", 1), // Number of miliseconds to wait between attepting @@ -194,7 +200,7 @@ // testing only. METASTOREFORCERELOADCONF("hive.metastore.force.reload.conf", false), METASTORESERVERMINTHREADS("hive.metastore.server.min.threads", 200), - METASTORESERVERMAXTHREADS("hive.metastore.server.max.threads", Integer.MAX_VALUE), + METASTORESERVERMAXTHREADS("hive.metastore.server.max.threads", 100000), METASTORE_TCP_KEEP_ALIVE("hive.metastore.server.tcp.keepalive", true), // Intermediate dir suffixes used for archiving. Not important what they // are, as long as collisions are avoided @@ -205,7 +211,8 @@ METASTORE_INT_EXTRACTED("hive.metastore.archive.intermediate.extracted", "_INTERMEDIATE_EXTRACTED"), METASTORE_KERBEROS_KEYTAB_FILE("hive.metastore.kerberos.keytab.file", ""), - METASTORE_KERBEROS_PRINCIPAL("hive.metastore.kerberos.principal", ""), + METASTORE_KERBEROS_PRINCIPAL("hive.metastore.kerberos.principal", + "hive-metastore/_HOST@EXAMPLE.COM"), METASTORE_USE_THRIFT_SASL("hive.metastore.sasl.enabled", false), METASTORE_CACHE_PINOBJTYPES("hive.metastore.cache.pinobjtypes", "Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order"), METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "DBCP"), @@ -228,7 +235,16 @@ METASTORE_EVENT_EXPIRY_DURATION("hive.metastore.event.expiry.duration",0L), METASTORE_MODE("hive.metastore.local",true), // Default parameters for creating tables - NEWTABLEDEFAULTPARA("hive.table.parameters.default",""), + NEWTABLEDEFAULTPARA("hive.table.parameters.default", ""), + METASTORE_RAW_STORE_IMPL("hive.metastore.rawstore.impl", + "org.apache.hadoop.hive.metastore.ObjectStore"), + METASTORE_CONNECTION_DRIVER("javax.jdo.option.ConnectionDriverName", + "org.apache.derby.jdbc.EmbeddedDriver"), + METASTORE_MANAGER_FACTORY_CLASS("javax.jdo.PersistenceManagerFactoryClass", + "org.datanucleus.jdo.JDOPersistenceManagerFactory"), + METASTORE_DETACH_ALL_ON_COMMIT("javax.jdo.option.DetachAllOnCommit", true), + METASTORE_NON_TRANSACTIONAL_READ("javax.jdo.option.NonTransactionalRead", true), + METASTORE_CONNECTION_USER_NAME("javax.jdo.option.ConnectionUserName", "APP"), // CLI CLIIGNOREERRORS("hive.cli.errors.ignore", false), @@ -336,7 +352,7 @@ HIVESKEWJOIN("hive.optimize.skewjoin", false), HIVECONVERTJOIN("hive.auto.convert.join", false), - HIVESKEWJOINKEY("hive.skewjoin.key", 1000000), + HIVESKEWJOINKEY("hive.skewjoin.key", 100000), HIVESKEWJOINMAPJOINNUMMAPTASK("hive.skewjoin.mapjoin.map.tasks", 10000), HIVESKEWJOINMAPJOINMINSPLIT("hive.skewjoin.mapjoin.min.split", 33554432L), //32M MAPREDMAXSPLITSIZE("mapred.max.split.size", 256000000L), @@ -426,7 +442,7 @@ HIVE_LOCK_MAPRED_ONLY("hive.lock.mapred.only.operation", false), HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", ""), - HIVE_ZOOKEEPER_CLIENT_PORT("hive.zookeeper.client.port", ""), + HIVE_ZOOKEEPER_CLIENT_PORT("hive.zookeeper.client.port", "2181"), HIVE_ZOOKEEPER_SESSION_TIMEOUT("hive.zookeeper.session.timeout", 600*1000), HIVE_ZOOKEEPER_NAMESPACE("hive.zookeeper.namespace", "hive_zookeeper_namespace"), HIVE_ZOOKEEPER_CLEAN_EXTRA_NODES("hive.zookeeper.clean.extra.nodes", false), @@ -452,13 +468,17 @@ SEMANTIC_ANALYZER_HOOK("hive.semantic.analyzer.hook",null), HIVE_AUTHORIZATION_ENABLED("hive.security.authorization.enabled", false), - HIVE_AUTHORIZATION_MANAGER("hive.security.authorization.manager", null), - HIVE_AUTHENTICATOR_MANAGER("hive.security.authenticator.manager", null), + HIVE_AUTHORIZATION_MANAGER("hive.security.authorization.manager", + "org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider"), + HIVE_AUTHENTICATOR_MANAGER("hive.security.authenticator.manager", + "org.apache.hadoop.hive.ql.security.HadoopDefaultAuthenticator"), + HIVE_AUTHORIZATION_TABLE_USER_GRANTS("hive.security.authorization.createtable.user.grants", ""), + HIVE_AUTHORIZATION_TABLE_GROUP_GRANTS("hive.security.authorization.createtable.group.grants", + ""), + HIVE_AUTHORIZATION_TABLE_ROLE_GRANTS("hive.security.authorization.createtable.role.grants", ""), + HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS("hive.security.authorization.createtable.owner.grants", + ""), - HIVE_AUTHORIZATION_TABLE_USER_GRANTS("hive.security.authorization.createtable.user.grants", null), - HIVE_AUTHORIZATION_TABLE_GROUP_GRANTS("hive.security.authorization.createtable.group.grants", null), - HIVE_AUTHORIZATION_TABLE_ROLE_GRANTS("hive.security.authorization.createtable.role.grants", null), - HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS("hive.security.authorization.createtable.owner.grants", null), // Print column names in output HIVE_CLI_PRINT_HEADER("hive.cli.print.header", false), @@ -473,6 +493,7 @@ HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS("hive.mapper.cannot.span.multiple.partitions", false), HIVE_REWORK_MAPREDWORK("hive.rework.mapredwork", false), HIVE_CONCATENATE_CHECK_INDEX ("hive.exec.concatenate.check.index", true), + HIVE_IO_EXCEPTION_HANDLERS("hive.io.exception.handlers", ""), //prefix used to auto generated column aliases HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL("hive.autogen.columnalias.prefix.label", "_c"), @@ -482,6 +503,7 @@ // The class responsible for logging client side performance metrics // Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger HIVE_PERF_LOGGER("hive.exec.perf.logger", "org.apache.hadoop.hive.ql.log.PerfLogger"), + HIVE_FS_HAR_IMPL("fs.har.impl", "org.apache.hadoop.hive.shims.HiveHarFileSystem"), // Whether to delete the scratchdir while startup HIVE_START_CLEANUP_SCRATCHDIR("hive.start.cleanup.scratchdir", false), HIVE_INSERT_INTO_MULTILEVEL_DIRS("hive.insert.into.multilevel.dirs", false), Index: conf/hive-default.xml =================================================================== --- conf/hive-default.xml (revision 1198626) +++ conf/hive-default.xml (working copy) @@ -379,13 +379,6 @@ - hive.optimize.pruner - true - Whether to enable the new partition pruner which depends on predicate pushdown. If this is disabled, - the old partition pruner which is based on AST will be enabled. - - - hive.optimize.groupby true Whether to enable the bucketed group by from bucketed partitions/tables. @@ -598,20 +591,7 @@ When the average output file size of a job is less than this number, Hive will start an additional map-reduce job to merge the output files into bigger files. This is only done for map-only jobs if hive.merge.mapfiles is true, and for map-reduce jobs if hive.merge.mapredfiles is true. - - hive.mapjoin.hashtable.initialCapacity - 100000 - In the process of Mapjoin, the key/value will be hold in the hashtable. This value means the initialCapacity of the in-memory hashtable - - - - hive.mapjoin.hashtable.loadfactor - 0.75 - In the process of Mapjoin, the key/value will be hold in the hashtable. This value means the load factor for the in-memory hashtable - - - hive.mapjoin.smalltable.filesize 25000000 The threshold for the input file size of the small tables; if the file size is smaller than this threshold, it will try to convert the common join into map join @@ -847,12 +827,6 @@ - hive.stats.jdbc.atomic - false - If this is set to true then the metastore stats will be updated only if all types of stats (# of rows, # of files, # of bytes etc.) are available. Otherwise metastore stats are updated in a best effort fashion with whatever are available. - - - hive.stats.retries.max 0 Maximum number of retries when stats publisher/aggregator got an exception updating intermediate database. Default is no tries on failures. @@ -871,12 +845,6 @@ - hive.concurrency.manager - org.apache.hadoop.hive.ql.lockmgr.ZooKeeperLockMgr - The concurrency manager for hive. - - - hive.lock.numretries 100 The number of times you want to try to get all the locks Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1198626) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy) @@ -239,7 +239,7 @@ } private boolean init() throws MetaException { - rawStoreClassName = hiveConf.get("hive.metastore.rawstore.impl"); + rawStoreClassName = hiveConf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL); checkForDefaultDb = hiveConf.getBoolean( "hive.metastore.checkForDefaultDb", true); String alterHandlerName = hiveConf.get("hive.metastore.alter.impl",