diff --git common/pom.xml common/pom.xml index b3d230d..776bb56 100644 --- common/pom.xml +++ common/pom.xml @@ -106,6 +106,12 @@ + + + ../conf/ + hive-default.xml.template + + ${basedir}/src/java ${basedir}/src/test ${basedir}/src/scripts diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index fa3e048..ca1fd86 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -24,28 +24,26 @@ import java.io.InputStream; import java.io.PrintStream; import java.net.URL; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Properties; -import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.security.auth.login.LoginException; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Shell; +import org.apache.hive.common.util.SystemVariables; +import org.w3c.dom.*; /** * Hive Configuration. @@ -60,7 +58,7 @@ private static URL hiveSiteURL = null; private static byte[] confVarByteArray = null; - private static final Map vars = new HashMap(); + private static final Map vars = new LinkedHashMap(); private final List restrictList = new ArrayList(); static { @@ -69,13 +67,36 @@ classLoader = HiveConf.class.getClassLoader(); } + for (ConfVars confVar : ConfVars.values()) { + vars.put(confVar.varname, confVar); + } + + ConfVars.HADOOPBIN.setValue(findHadoopBinary()); + ConfVars.HADOOPBIN.setDescription( + " Hadoop Configuration Properties. \n" + + " Properties with null values are ignored and exist only for the purpose of giving us a \n" + + " symbolic name to reference in the Hive source code. Properties with non-null values \n" + + " will override any values set in the underlying Hadoop configuration."); + + registerConfVars(classLoader.getResource("hive-default.xml.template")); + hiveDefaultURL = classLoader.getResource("hive-default.xml"); // Look for hive-site.xml on the CLASSPATH and log its location if found. hiveSiteURL = classLoader.getResource("hive-site.xml"); - for (ConfVars confVar : ConfVars.values()) { - vars.put(confVar.varname, confVar); + } + + private static String findHadoopBinary() { + String val = System.getenv("HADOOP_HOME"); + // In Hadoop 1.X and Hadoop 2.X HADOOP_HOME is gone and replaced with HADOOP_PREFIX + if (val == null) { + val = System.getenv("HADOOP_PREFIX"); } + // and if all else fails we can at least try /usr/bin/hadoop + val = (val == null ? File.separator + "usr" : val) + + File.separator + "bin" + File.separator + "hadoop"; + // Launch hadoop command file on windows. + return val + (Shell.WINDOWS ? ".cmd" : ""); } /** @@ -167,792 +188,717 @@ */ public static enum ConfVars { // QL execution stuff - SCRIPTWRAPPER("hive.exec.script.wrapper", null), - PLAN("hive.exec.plan", ""), - PLAN_SERIALIZATION("hive.plan.serialization.format","kryo"), - SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive-" + System.getProperty("user.name")), - LOCALSCRATCHDIR("hive.exec.local.scratchdir", System.getProperty("java.io.tmpdir") + File.separator + System.getProperty("user.name")), - SCRATCHDIRPERMISSION("hive.scratch.dir.permission", "700"), - SUBMITVIACHILD("hive.exec.submitviachild", false), - SCRIPTERRORLIMIT("hive.exec.script.maxerrsize", 100000), - ALLOWPARTIALCONSUMP("hive.exec.script.allow.partial.consumption", false), - STREAMREPORTERPERFIX("stream.stderr.reporter.prefix", "reporter:"), - STREAMREPORTERENABLED("stream.stderr.reporter.enabled", true), - COMPRESSRESULT("hive.exec.compress.output", false), - COMPRESSINTERMEDIATE("hive.exec.compress.intermediate", false), - COMPRESSINTERMEDIATECODEC("hive.intermediate.compression.codec", ""), - COMPRESSINTERMEDIATETYPE("hive.intermediate.compression.type", ""), - BYTESPERREDUCER("hive.exec.reducers.bytes.per.reducer", (long) (1000 * 1000 * 1000)), - MAXREDUCERS("hive.exec.reducers.max", 999), - PREEXECHOOKS("hive.exec.pre.hooks", ""), - POSTEXECHOOKS("hive.exec.post.hooks", ""), - ONFAILUREHOOKS("hive.exec.failure.hooks", ""), - CLIENTSTATSPUBLISHERS("hive.client.stats.publishers", ""), - EXECPARALLEL("hive.exec.parallel", false), // parallel query launching - EXECPARALLETHREADNUMBER("hive.exec.parallel.thread.number", 8), - HIVESPECULATIVEEXECREDUCERS("hive.mapred.reduce.tasks.speculative.execution", true), - HIVECOUNTERSPULLINTERVAL("hive.exec.counters.pull.interval", 1000L), - DYNAMICPARTITIONING("hive.exec.dynamic.partition", true), - DYNAMICPARTITIONINGMODE("hive.exec.dynamic.partition.mode", "strict"), - DYNAMICPARTITIONMAXPARTS("hive.exec.max.dynamic.partitions", 1000), - DYNAMICPARTITIONMAXPARTSPERNODE("hive.exec.max.dynamic.partitions.pernode", 100), - MAXCREATEDFILES("hive.exec.max.created.files", 100000L), - DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir", - System.getProperty("java.io.tmpdir") + File.separator + "${hive.session.id}_resources"), - DEFAULTPARTITIONNAME("hive.exec.default.partition.name", "__HIVE_DEFAULT_PARTITION__"), - DEFAULT_ZOOKEEPER_PARTITION_NAME("hive.lockmgr.zookeeper.default.partition.name", "__HIVE_DEFAULT_ZOOKEEPER_PARTITION__"), + SCRIPTWRAPPER("hive.exec.script.wrapper"), + PLAN("hive.exec.plan", true), + PLAN_SERIALIZATION("hive.plan.serialization.format"), + SCRATCHDIR("hive.exec.scratchdir"), + LOCALSCRATCHDIR("hive.exec.local.scratchdir"), + SCRATCHDIRPERMISSION("hive.scratch.dir.permission"), + SUBMITVIACHILD("hive.exec.submitviachild", VarType.BOOLEAN), + SCRIPTERRORLIMIT("hive.exec.script.maxerrsize", VarType.INT), + ALLOWPARTIALCONSUMP("hive.exec.script.allow.partial.consumption", VarType.BOOLEAN), + STREAMREPORTERPERFIX("stream.stderr.reporter.prefix"), + STREAMREPORTERENABLED("stream.stderr.reporter.enabled", VarType.BOOLEAN), + COMPRESSRESULT("hive.exec.compress.output", VarType.BOOLEAN), + COMPRESSINTERMEDIATE("hive.exec.compress.intermediate", VarType.BOOLEAN), + COMPRESSINTERMEDIATECODEC("hive.intermediate.compression.codec", true), + COMPRESSINTERMEDIATETYPE("hive.intermediate.compression.type", true), + BYTESPERREDUCER("hive.exec.reducers.bytes.per.reducer", VarType.LONG), + MAXREDUCERS("hive.exec.reducers.max", VarType.INT), + PREEXECHOOKS("hive.exec.pre.hooks", true), + POSTEXECHOOKS("hive.exec.post.hooks", true), + ONFAILUREHOOKS("hive.exec.failure.hooks", true), + CLIENTSTATSPUBLISHERS("hive.client.stats.publishers", true), + EXECPARALLEL("hive.exec.parallel", VarType.BOOLEAN), // parallel query launching + EXECPARALLETHREADNUMBER("hive.exec.parallel.thread.number", VarType.INT), + HIVESPECULATIVEEXECREDUCERS("hive.mapred.reduce.tasks.speculative.execution", VarType.BOOLEAN), + HIVECOUNTERSPULLINTERVAL("hive.exec.counters.pull.interval", VarType.LONG), + DYNAMICPARTITIONING("hive.exec.dynamic.partition", VarType.BOOLEAN), + DYNAMICPARTITIONINGMODE("hive.exec.dynamic.partition.mode"), + DYNAMICPARTITIONMAXPARTS("hive.exec.max.dynamic.partitions", VarType.INT), + DYNAMICPARTITIONMAXPARTSPERNODE("hive.exec.max.dynamic.partitions.pernode", VarType.INT), + MAXCREATEDFILES("hive.exec.max.created.files", VarType.LONG), + DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir"), + DEFAULTPARTITIONNAME("hive.exec.default.partition.name"), + DEFAULT_ZOOKEEPER_PARTITION_NAME("hive.lockmgr.zookeeper.default.partition.name"), // Whether to show a link to the most failed task + debugging tips - SHOW_JOB_FAIL_DEBUG_INFO("hive.exec.show.job.failure.debug.info", true), - JOB_DEBUG_CAPTURE_STACKTRACES("hive.exec.job.debug.capture.stacktraces", true), - JOB_DEBUG_TIMEOUT("hive.exec.job.debug.timeout", 30000), - TASKLOG_DEBUG_TIMEOUT("hive.exec.tasklog.debug.timeout", 20000), - OUTPUT_FILE_EXTENSION("hive.output.file.extension", null), + SHOW_JOB_FAIL_DEBUG_INFO("hive.exec.show.job.failure.debug.info", VarType.BOOLEAN), + JOB_DEBUG_CAPTURE_STACKTRACES("hive.exec.job.debug.capture.stacktraces", VarType.BOOLEAN), + JOB_DEBUG_TIMEOUT("hive.exec.job.debug.timeout", VarType.INT), + TASKLOG_DEBUG_TIMEOUT("hive.exec.tasklog.debug.timeout", VarType.INT), + OUTPUT_FILE_EXTENSION("hive.output.file.extension"), // should hive determine whether to run in local mode automatically ? - LOCALMODEAUTO("hive.exec.mode.local.auto", false), + LOCALMODEAUTO("hive.exec.mode.local.auto", VarType.BOOLEAN), // if yes: // run in local mode only if input bytes is less than this. 128MB by default - LOCALMODEMAXBYTES("hive.exec.mode.local.auto.inputbytes.max", 134217728L), + LOCALMODEMAXBYTES("hive.exec.mode.local.auto.inputbytes.max", VarType.LONG), // run in local mode only if number of tasks (for map and reduce each) is // less than this - LOCALMODEMAXINPUTFILES("hive.exec.mode.local.auto.input.files.max", 4), + LOCALMODEMAXINPUTFILES("hive.exec.mode.local.auto.input.files.max", VarType.INT), // if true, DROP TABLE/VIEW does not fail if table/view doesn't exist and IF EXISTS is // not specified - DROPIGNORESNONEXISTENT("hive.exec.drop.ignorenonexistent", true), + DROPIGNORESNONEXISTENT("hive.exec.drop.ignorenonexistent", VarType.BOOLEAN), // ignore the mapjoin hint - HIVEIGNOREMAPJOINHINT("hive.ignore.mapjoin.hint", true), + HIVEIGNOREMAPJOINHINT("hive.ignore.mapjoin.hint", VarType.BOOLEAN), // Hadoop Configuration Properties // Properties with null values are ignored and exist only for the purpose of giving us // a symbolic name to reference in the Hive source code. Properties with non-null // values will override any values set in the underlying Hadoop configuration. - HADOOPBIN("hadoop.bin.path", findHadoopBinary()), - HADOOPFS("fs.default.name", null), - HIVE_FS_HAR_IMPL("fs.har.impl", "org.apache.hadoop.hive.shims.HiveHarFileSystem"), - HADOOPMAPFILENAME("map.input.file", null), - HADOOPMAPREDINPUTDIR("mapred.input.dir", null), - HADOOPMAPREDINPUTDIRRECURSIVE("mapred.input.dir.recursive", false), - HADOOPJT("mapred.job.tracker", null), - MAPREDMAXSPLITSIZE("mapred.max.split.size", 256000000L), - MAPREDMINSPLITSIZE("mapred.min.split.size", 1L), - MAPREDMINSPLITSIZEPERNODE("mapred.min.split.size.per.rack", 1L), - MAPREDMINSPLITSIZEPERRACK("mapred.min.split.size.per.node", 1L), + HADOOPBIN("hadoop.bin.path"), + HADOOPFS("fs.default.name"), + HIVE_FS_HAR_IMPL("fs.har.impl"), + HADOOPMAPFILENAME("map.input.file"), + HADOOPMAPREDINPUTDIR("mapred.input.dir"), + HADOOPMAPREDINPUTDIRRECURSIVE("mapred.input.dir.recursive", VarType.BOOLEAN), + HADOOPJT("mapred.job.tracker"), + MAPREDMAXSPLITSIZE("mapred.max.split.size", VarType.LONG), + MAPREDMINSPLITSIZE("mapred.min.split.size", VarType.LONG), + MAPREDMINSPLITSIZEPERNODE("mapred.min.split.size.per.rack", VarType.LONG), + MAPREDMINSPLITSIZEPERRACK("mapred.min.split.size.per.node", VarType.LONG), // The number of reduce tasks per job. Hadoop sets this value to 1 by default // By setting this property to -1, Hive will automatically determine the correct // number of reducers. - HADOOPNUMREDUCERS("mapred.reduce.tasks", -1), - HADOOPJOBNAME("mapred.job.name", null), - HADOOPSPECULATIVEEXECREDUCERS("mapred.reduce.tasks.speculative.execution", true), + HADOOPNUMREDUCERS("mapred.reduce.tasks", VarType.INT), + HADOOPJOBNAME("mapred.job.name"), + HADOOPSPECULATIVEEXECREDUCERS("mapred.reduce.tasks.speculative.execution", VarType.BOOLEAN), // Metastore stuff. Be sure to update HiveConf.metaVars when you add // something here! - METASTOREDIRECTORY("hive.metastore.metadb.dir", ""), - METASTOREWAREHOUSE("hive.metastore.warehouse.dir", "/user/hive/warehouse"), - METASTOREURIS("hive.metastore.uris", ""), + METASTOREDIRECTORY("hive.metastore.metadb.dir", true), + METASTOREWAREHOUSE("hive.metastore.warehouse.dir", true), + METASTOREURIS("hive.metastore.uris", true), // Number of times to retry a connection to a Thrift metastore server - METASTORETHRIFTCONNECTIONRETRIES("hive.metastore.connect.retries", 3), + METASTORETHRIFTCONNECTIONRETRIES("hive.metastore.connect.retries", VarType.INT), // Number of times to retry a Thrift metastore call upon failure - METASTORETHRIFTFAILURERETRIES("hive.metastore.failure.retries", 1), + METASTORETHRIFTFAILURERETRIES("hive.metastore.failure.retries", VarType.INT), // Number of seconds the client should wait between connection attempts - METASTORE_CLIENT_CONNECT_RETRY_DELAY("hive.metastore.client.connect.retry.delay", 1), + METASTORE_CLIENT_CONNECT_RETRY_DELAY("hive.metastore.client.connect.retry.delay", VarType.INT), // Socket timeout for the client connection (in seconds) - METASTORE_CLIENT_SOCKET_TIMEOUT("hive.metastore.client.socket.timeout", 20), - METASTOREPWD("javax.jdo.option.ConnectionPassword", "mine"), + METASTORE_CLIENT_SOCKET_TIMEOUT("hive.metastore.client.socket.timeout", VarType.INT), + METASTOREPWD("javax.jdo.option.ConnectionPassword"), // Class name of JDO connection url hook - METASTORECONNECTURLHOOK("hive.metastore.ds.connection.url.hook", ""), - METASTOREMULTITHREADED("javax.jdo.option.Multithreaded", true), + METASTORECONNECTURLHOOK("hive.metastore.ds.connection.url.hook"), + METASTOREMULTITHREADED("javax.jdo.option.Multithreaded", VarType.BOOLEAN), // Name of the connection url in the configuration - METASTORECONNECTURLKEY("javax.jdo.option.ConnectionURL", - "jdbc:derby:;databaseName=metastore_db;create=true"), + METASTORECONNECTURLKEY("javax.jdo.option.ConnectionURL", true), // Number of attempts to retry connecting after there is a JDO datastore err - METASTOREATTEMPTS("hive.metastore.ds.retry.attempts", 1), + METASTOREATTEMPTS("hive.metastore.ds.retry.attempts", VarType.INT), // Number of miliseconds to wait between attepting - METASTOREINTERVAL("hive.metastore.ds.retry.interval", 1000), + METASTOREINTERVAL("hive.metastore.ds.retry.interval", VarType.INT), // Whether to force reloading of the metastore configuration (including // the connection URL, before the next metastore query that accesses the // datastore. Once reloaded, this value is reset to false. Used for // testing only. - METASTOREFORCERELOADCONF("hive.metastore.force.reload.conf", false), + METASTOREFORCERELOADCONF("hive.metastore.force.reload.conf", VarType.BOOLEAN), // Number of attempts to retry connecting after there is a JDO datastore err - HMSHANDLERATTEMPTS("hive.hmshandler.retry.attempts", 1), + HMSHANDLERATTEMPTS("hive.hmshandler.retry.attempts", VarType.INT), // Number of miliseconds to wait between attepting - HMSHANDLERINTERVAL("hive.hmshandler.retry.interval", 1000), + HMSHANDLERINTERVAL("hive.hmshandler.retry.interval", VarType.INT), // Whether to force reloading of the HMSHandler configuration (including // the connection URL, before the next metastore query that accesses the // datastore. Once reloaded, this value is reset to false. Used for // testing only. - HMSHANDLERFORCERELOADCONF("hive.hmshandler.force.reload.conf", false), - METASTORESERVERMINTHREADS("hive.metastore.server.min.threads", 200), - METASTORESERVERMAXTHREADS("hive.metastore.server.max.threads", 100000), - METASTORE_TCP_KEEP_ALIVE("hive.metastore.server.tcp.keepalive", true), + HMSHANDLERFORCERELOADCONF("hive.hmshandler.force.reload.conf", VarType.BOOLEAN), + METASTORESERVERMINTHREADS("hive.metastore.server.min.threads", VarType.INT), + METASTORESERVERMAXTHREADS("hive.metastore.server.max.threads", VarType.INT), + METASTORE_TCP_KEEP_ALIVE("hive.metastore.server.tcp.keepalive", VarType.BOOLEAN), // Intermediate dir suffixes used for archiving. Not important what they // are, as long as collisions are avoided - METASTORE_INT_ORIGINAL("hive.metastore.archive.intermediate.original", - "_INTERMEDIATE_ORIGINAL"), - METASTORE_INT_ARCHIVED("hive.metastore.archive.intermediate.archived", - "_INTERMEDIATE_ARCHIVED"), - METASTORE_INT_EXTRACTED("hive.metastore.archive.intermediate.extracted", - "_INTERMEDIATE_EXTRACTED"), - METASTORE_KERBEROS_KEYTAB_FILE("hive.metastore.kerberos.keytab.file", ""), - METASTORE_KERBEROS_PRINCIPAL("hive.metastore.kerberos.principal", - "hive-metastore/_HOST@EXAMPLE.COM"), - METASTORE_USE_THRIFT_SASL("hive.metastore.sasl.enabled", false), - METASTORE_USE_THRIFT_FRAMED_TRANSPORT("hive.metastore.thrift.framed.transport.enabled", false), + METASTORE_INT_ORIGINAL("hive.metastore.archive.intermediate.original"), + METASTORE_INT_ARCHIVED("hive.metastore.archive.intermediate.archived"), + METASTORE_INT_EXTRACTED("hive.metastore.archive.intermediate.extracted"), + METASTORE_KERBEROS_KEYTAB_FILE("hive.metastore.kerberos.keytab.file", true), + METASTORE_KERBEROS_PRINCIPAL("hive.metastore.kerberos.principal"), + METASTORE_USE_THRIFT_SASL("hive.metastore.sasl.enabled", VarType.BOOLEAN), + METASTORE_USE_THRIFT_FRAMED_TRANSPORT("hive.metastore.thrift.framed.transport.enabled", VarType.BOOLEAN), METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_CLS( - "hive.cluster.delegation.token.store.class", - "org.apache.hadoop.hive.thrift.MemoryTokenStore"), + "hive.cluster.delegation.token.store.class"), METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_ZK_CONNECTSTR( - "hive.cluster.delegation.token.store.zookeeper.connectString", ""), + "hive.cluster.delegation.token.store.zookeeper.connectString", true), METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_ZK_ZNODE( - "hive.cluster.delegation.token.store.zookeeper.znode", "/hive/cluster/delegation"), + "hive.cluster.delegation.token.store.zookeeper.znode"), METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_ZK_ACL( - "hive.cluster.delegation.token.store.zookeeper.acl", ""), - METASTORE_CACHE_PINOBJTYPES("hive.metastore.cache.pinobjtypes", "Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order"), - METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "BONECP"), - METASTORE_VALIDATE_TABLES("datanucleus.validateTables", false), - METASTORE_VALIDATE_COLUMNS("datanucleus.validateColumns", false), - METASTORE_VALIDATE_CONSTRAINTS("datanucleus.validateConstraints", false), - METASTORE_STORE_MANAGER_TYPE("datanucleus.storeManagerType", "rdbms"), - METASTORE_AUTO_CREATE_SCHEMA("datanucleus.autoCreateSchema", true), - METASTORE_FIXED_DATASTORE("datanucleus.fixedDatastore", false), - METASTORE_SCHEMA_VERIFICATION("hive.metastore.schema.verification", false), - METASTORE_AUTO_START_MECHANISM_MODE("datanucleus.autoStartMechanismMode", "checked"), - METASTORE_TRANSACTION_ISOLATION("datanucleus.transactionIsolation", "read-committed"), - METASTORE_CACHE_LEVEL2("datanucleus.cache.level2", false), - METASTORE_CACHE_LEVEL2_TYPE("datanucleus.cache.level2.type", "none"), - METASTORE_IDENTIFIER_FACTORY("datanucleus.identifierFactory", "datanucleus1"), - METASTORE_USE_LEGACY_VALUE_STRATEGY("datanucleus.rdbms.useLegacyNativeValueStrategy", true), - METASTORE_PLUGIN_REGISTRY_BUNDLE_CHECK("datanucleus.plugin.pluginRegistryBundleCheck", "LOG"), - METASTORE_BATCH_RETRIEVE_MAX("hive.metastore.batch.retrieve.max", 300), + "hive.cluster.delegation.token.store.zookeeper.acl", true), + METASTORE_CACHE_PINOBJTYPES("hive.metastore.cache.pinobjtypes"), + METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType"), + METASTORE_VALIDATE_TABLES("datanucleus.validateTables", VarType.BOOLEAN), + METASTORE_VALIDATE_COLUMNS("datanucleus.validateColumns", VarType.BOOLEAN), + METASTORE_VALIDATE_CONSTRAINTS("datanucleus.validateConstraints", VarType.BOOLEAN), + METASTORE_STORE_MANAGER_TYPE("datanucleus.storeManagerType"), + METASTORE_AUTO_CREATE_SCHEMA("datanucleus.autoCreateSchema", VarType.BOOLEAN), + METASTORE_FIXED_DATASTORE("datanucleus.fixedDatastore", VarType.BOOLEAN), + METASTORE_SCHEMA_VERIFICATION("hive.metastore.schema.verification", VarType.BOOLEAN), + METASTORE_AUTO_START_MECHANISM_MODE("datanucleus.autoStartMechanismMode"), + METASTORE_TRANSACTION_ISOLATION("datanucleus.transactionIsolation"), + METASTORE_CACHE_LEVEL2("datanucleus.cache.level2", VarType.BOOLEAN), + METASTORE_CACHE_LEVEL2_TYPE("datanucleus.cache.level2.type"), + METASTORE_IDENTIFIER_FACTORY("datanucleus.identifierFactory"), + METASTORE_USE_LEGACY_VALUE_STRATEGY("datanucleus.rdbms.useLegacyNativeValueStrategy", VarType.BOOLEAN), + METASTORE_PLUGIN_REGISTRY_BUNDLE_CHECK("datanucleus.plugin.pluginRegistryBundleCheck"), + METASTORE_BATCH_RETRIEVE_MAX("hive.metastore.batch.retrieve.max", VarType.INT), METASTORE_BATCH_RETRIEVE_TABLE_PARTITION_MAX( - "hive.metastore.batch.retrieve.table.partition.max", 1000), + "hive.metastore.batch.retrieve.table.partition.max", VarType.INT), // A comma separated list of hooks which implement MetaStoreInitListener and will be run at // the beginning of HMSHandler initialization - METASTORE_INIT_HOOKS("hive.metastore.init.hooks", ""), - METASTORE_PRE_EVENT_LISTENERS("hive.metastore.pre.event.listeners", ""), - METASTORE_EVENT_LISTENERS("hive.metastore.event.listeners", ""), + METASTORE_INIT_HOOKS("hive.metastore.init.hooks", true), + METASTORE_PRE_EVENT_LISTENERS("hive.metastore.pre.event.listeners", true), + METASTORE_EVENT_LISTENERS("hive.metastore.event.listeners", true), // should we do checks against the storage (usually hdfs) for operations like drop_partition - METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS("hive.metastore.authorization.storage.checks", false), - METASTORE_EVENT_CLEAN_FREQ("hive.metastore.event.clean.freq",0L), - METASTORE_EVENT_EXPIRY_DURATION("hive.metastore.event.expiry.duration",0L), - METASTORE_EXECUTE_SET_UGI("hive.metastore.execute.setugi", false), + METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS("hive.metastore.authorization.storage.checks", VarType.BOOLEAN), + METASTORE_EVENT_CLEAN_FREQ("hive.metastore.event.clean.freq",VarType.LONG), + METASTORE_EVENT_EXPIRY_DURATION("hive.metastore.event.expiry.duration",VarType.LONG), + METASTORE_EXECUTE_SET_UGI("hive.metastore.execute.setugi", VarType.BOOLEAN), METASTORE_PARTITION_NAME_WHITELIST_PATTERN( - "hive.metastore.partition.name.whitelist.pattern", ""), - METASTORE_TRY_DIRECT_SQL("hive.metastore.try.direct.sql", true), - METASTORE_TRY_DIRECT_SQL_DDL("hive.metastore.try.direct.sql.ddl", true), + "hive.metastore.partition.name.whitelist.pattern", true), + METASTORE_TRY_DIRECT_SQL("hive.metastore.try.direct.sql", VarType.BOOLEAN), + METASTORE_TRY_DIRECT_SQL_DDL("hive.metastore.try.direct.sql.ddl", VarType.BOOLEAN), METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES( - "hive.metastore.disallow.incompatible.col.type.changes", false), + "hive.metastore.disallow.incompatible.col.type.change", VarType.BOOLEAN), // Default parameters for creating tables - NEWTABLEDEFAULTPARA("hive.table.parameters.default", ""), + NEWTABLEDEFAULTPARA("hive.table.parameters.default", true), // Parameters to copy over when creating a table with Create Table Like. - DDL_CTL_PARAMETERS_WHITELIST("hive.ddl.createtablelike.properties.whitelist", ""), - METASTORE_RAW_STORE_IMPL("hive.metastore.rawstore.impl", - "org.apache.hadoop.hive.metastore.ObjectStore"), - METASTORE_CONNECTION_DRIVER("javax.jdo.option.ConnectionDriverName", - "org.apache.derby.jdbc.EmbeddedDriver"), - METASTORE_MANAGER_FACTORY_CLASS("javax.jdo.PersistenceManagerFactoryClass", - "org.datanucleus.api.jdo.JDOPersistenceManagerFactory"), - METASTORE_EXPRESSION_PROXY_CLASS("hive.metastore.expression.proxy", - "org.apache.hadoop.hive.ql.optimizer.ppr.PartitionExpressionForMetastore"), - METASTORE_DETACH_ALL_ON_COMMIT("javax.jdo.option.DetachAllOnCommit", true), - METASTORE_NON_TRANSACTIONAL_READ("javax.jdo.option.NonTransactionalRead", true), - METASTORE_CONNECTION_USER_NAME("javax.jdo.option.ConnectionUserName", "APP"), - METASTORE_END_FUNCTION_LISTENERS("hive.metastore.end.function.listeners", ""), - METASTORE_PART_INHERIT_TBL_PROPS("hive.metastore.partition.inherit.table.properties",""), + DDL_CTL_PARAMETERS_WHITELIST("hive.ddl.createtablelike.properties.whitelist", true), + METASTORE_RAW_STORE_IMPL("hive.metastore.rawstore.impl"), + METASTORE_CONNECTION_DRIVER("javax.jdo.option.ConnectionDriverName"), + METASTORE_MANAGER_FACTORY_CLASS("javax.jdo.PersistenceManagerFactoryClass"), + METASTORE_EXPRESSION_PROXY_CLASS("hive.metastore.expression.proxy"), + METASTORE_DETACH_ALL_ON_COMMIT("javax.jdo.option.DetachAllOnCommit", VarType.BOOLEAN), + METASTORE_NON_TRANSACTIONAL_READ("javax.jdo.option.NonTransactionalRead", VarType.BOOLEAN), + METASTORE_CONNECTION_USER_NAME("javax.jdo.option.ConnectionUserName"), + METASTORE_END_FUNCTION_LISTENERS("hive.metastore.end.function.listeners", true), + METASTORE_PART_INHERIT_TBL_PROPS("hive.metastore.partition.inherit.table.properties", true), // Parameters for exporting metadata on table drop (requires the use of the) // org.apache.hadoop.hive.ql.parse.MetaDataExportListener preevent listener - METADATA_EXPORT_LOCATION("hive.metadata.export.location", ""), - MOVE_EXPORTED_METADATA_TO_TRASH("hive.metadata.move.exported.metadata.to.trash", true), + METADATA_EXPORT_LOCATION("hive.metadata.export.location", true), + MOVE_EXPORTED_METADATA_TO_TRASH("hive.metadata.move.exported.metadata.to.trash", VarType.BOOLEAN), // CLI - CLIIGNOREERRORS("hive.cli.errors.ignore", false), - CLIPRINTCURRENTDB("hive.cli.print.current.db", false), - CLIPROMPT("hive.cli.prompt", "hive"), - CLIPRETTYOUTPUTNUMCOLS("hive.cli.pretty.output.num.cols", -1), + CLIIGNOREERRORS("hive.cli.errors.ignore", VarType.BOOLEAN), + CLIPRINTCURRENTDB("hive.cli.print.current.db", VarType.BOOLEAN), + CLIPROMPT("hive.cli.prompt"), + CLIPRETTYOUTPUTNUMCOLS("hive.cli.pretty.output.num.cols", VarType.INT), - HIVE_METASTORE_FS_HANDLER_CLS("hive.metastore.fs.handler.class", "org.apache.hadoop.hive.metastore.HiveMetaStoreFsImpl"), + HIVE_METASTORE_FS_HANDLER_CLS("hive.metastore.fs.handler.class"), // Things we log in the jobconf // session identifier - HIVESESSIONID("hive.session.id", ""), + HIVESESSIONID("hive.session.id", true), // whether session is running in silent mode or not - HIVESESSIONSILENT("hive.session.silent", false), + HIVESESSIONSILENT("hive.session.silent", VarType.BOOLEAN), // Whether to enable history for this session - HIVE_SESSION_HISTORY_ENABLED("hive.session.history.enabled", false), + HIVE_SESSION_HISTORY_ENABLED("hive.session.history.enabled", VarType.BOOLEAN), // query being executed (multiple per session) - HIVEQUERYSTRING("hive.query.string", ""), + HIVEQUERYSTRING("hive.query.string", true), // id of query being executed (multiple per session) - HIVEQUERYID("hive.query.id", ""), + HIVEQUERYID("hive.query.id", true), // id of the mapred plan being executed (multiple per query) - HIVEPLANID("hive.query.planid", ""), + HIVEPLANID("hive.query.planid", true), // max jobname length - HIVEJOBNAMELENGTH("hive.jobname.length", 50), + HIVEJOBNAMELENGTH("hive.jobname.length", VarType.INT), // hive jar - HIVEJAR("hive.jar.path", ""), - HIVEAUXJARS("hive.aux.jars.path", ""), + HIVEJAR("hive.jar.path", true), + HIVEAUXJARS("hive.aux.jars.path", true), // hive added files and jars - HIVEADDEDFILES("hive.added.files.path", ""), - HIVEADDEDJARS("hive.added.jars.path", ""), - HIVEADDEDARCHIVES("hive.added.archives.path", ""), + HIVEADDEDFILES("hive.added.files.path", true), + HIVEADDEDJARS("hive.added.jars.path", true), + HIVEADDEDARCHIVES("hive.added.archives.path", true), // for hive script operator - HIVES_AUTO_PROGRESS_TIMEOUT("hive.auto.progress.timeout", 0), - HIVETABLENAME("hive.table.name", ""), - HIVEPARTITIONNAME("hive.partition.name", ""), - HIVESCRIPTAUTOPROGRESS("hive.script.auto.progress", false), - HIVESCRIPTIDENVVAR("hive.script.operator.id.env.var", "HIVE_SCRIPT_OPERATOR_ID"), - HIVESCRIPTTRUNCATEENV("hive.script.operator.truncate.env", false), - HIVEMAPREDMODE("hive.mapred.mode", "nonstrict"), - HIVEALIAS("hive.alias", ""), - HIVEMAPSIDEAGGREGATE("hive.map.aggr", true), - HIVEGROUPBYSKEW("hive.groupby.skewindata", false), - HIVE_OPTIMIZE_MULTI_GROUPBY_COMMON_DISTINCTS("hive.optimize.multigroupby.common.distincts", - true), - HIVEJOINEMITINTERVAL("hive.join.emit.interval", 1000), - HIVEJOINCACHESIZE("hive.join.cache.size", 25000), + HIVES_AUTO_PROGRESS_TIMEOUT("hive.auto.progress.timeout", VarType.INT), + HIVETABLENAME("hive.table.name", true), + HIVEPARTITIONNAME("hive.partition.name", true), + HIVESCRIPTAUTOPROGRESS("hive.script.auto.progress", VarType.BOOLEAN), + HIVESCRIPTIDENVVAR("hive.script.operator.id.env.var"), + HIVESCRIPTTRUNCATEENV("hive.script.operator.truncate.env", VarType.BOOLEAN), + HIVEMAPREDMODE("hive.mapred.mode"), + HIVEALIAS("hive.alias", true), + HIVEMAPSIDEAGGREGATE("hive.map.aggr", VarType.BOOLEAN), + HIVEGROUPBYSKEW("hive.groupby.skewindata", VarType.BOOLEAN), + HIVE_OPTIMIZE_MULTI_GROUPBY_COMMON_DISTINCTS("hive.optimize.multigroupby.common.distincts", VarType.BOOLEAN), + HIVEJOINEMITINTERVAL("hive.join.emit.interval", VarType.INT), + HIVEJOINCACHESIZE("hive.join.cache.size", VarType.INT), // hive.mapjoin.bucket.cache.size has been replaced by hive.smbjoin.cache.row, // need to remove by hive .13. Also, do not change default (see SMB operator) - HIVEMAPJOINBUCKETCACHESIZE("hive.mapjoin.bucket.cache.size", 100), - - HIVESMBJOINCACHEROWS("hive.smbjoin.cache.rows", 10000), - HIVEGROUPBYMAPINTERVAL("hive.groupby.mapaggr.checkinterval", 100000), - HIVEMAPAGGRHASHMEMORY("hive.map.aggr.hash.percentmemory", (float) 0.5), - HIVEMAPJOINFOLLOWEDBYMAPAGGRHASHMEMORY("hive.mapjoin.followby.map.aggr.hash.percentmemory", (float) 0.3), - HIVEMAPAGGRMEMORYTHRESHOLD("hive.map.aggr.hash.force.flush.memory.threshold", (float) 0.9), - HIVEMAPAGGRHASHMINREDUCTION("hive.map.aggr.hash.min.reduction", (float) 0.5), - HIVEMULTIGROUPBYSINGLEREDUCER("hive.multigroupby.singlereducer", true), - HIVE_MAP_GROUPBY_SORT("hive.map.groupby.sorted", false), - HIVE_MAP_GROUPBY_SORT_TESTMODE("hive.map.groupby.sorted.testmode", false), - HIVE_GROUPBY_ORDERBY_POSITION_ALIAS("hive.groupby.orderby.position.alias", false), - HIVE_NEW_JOB_GROUPING_SET_CARDINALITY("hive.new.job.grouping.set.cardinality", 30), + HIVEMAPJOINBUCKETCACHESIZE("hive.mapjoin.bucket.cache.size", VarType.INT), + + HIVESMBJOINCACHEROWS("hive.smbjoin.cache.rows", VarType.INT), + HIVEGROUPBYMAPINTERVAL("hive.groupby.mapaggr.checkinterval", VarType.INT), + HIVEMAPAGGRHASHMEMORY("hive.map.aggr.hash.percentmemory", VarType.FLOAT), + HIVEMAPJOINFOLLOWEDBYMAPAGGRHASHMEMORY("hive.mapjoin.followby.map.aggr.hash.percentmemory", VarType.FLOAT), + HIVEMAPAGGRMEMORYTHRESHOLD("hive.map.aggr.hash.force.flush.memory.threshold", VarType.FLOAT), + HIVEMAPAGGRHASHMINREDUCTION("hive.map.aggr.hash.min.reduction", VarType.FLOAT), + HIVEMULTIGROUPBYSINGLEREDUCER("hive.multigroupby.singlereducer", VarType.BOOLEAN), + HIVE_MAP_GROUPBY_SORT("hive.map.groupby.sorted", VarType.BOOLEAN), + HIVE_MAP_GROUPBY_SORT_TESTMODE("hive.map.groupby.sorted.testmode", VarType.BOOLEAN), + HIVE_GROUPBY_ORDERBY_POSITION_ALIAS("hive.groupby.orderby.position.alias", VarType.BOOLEAN), + HIVE_NEW_JOB_GROUPING_SET_CARDINALITY("hive.new.job.grouping.set.cardinality", VarType.INT), // for hive udtf operator - HIVEUDTFAUTOPROGRESS("hive.udtf.auto.progress", false), + HIVEUDTFAUTOPROGRESS("hive.udtf.auto.progress", VarType.BOOLEAN), // Default file format for CREATE TABLE statement // Options: TextFile, SequenceFile - HIVEDEFAULTFILEFORMAT("hive.default.fileformat", "TextFile", - new StringsValidator("TextFile", "SequenceFile", "RCfile", "ORC")), - HIVEQUERYRESULTFILEFORMAT("hive.query.result.fileformat", "TextFile", - new StringsValidator("TextFile", "SequenceFile", "RCfile")), - HIVECHECKFILEFORMAT("hive.fileformat.check", true), + HIVEDEFAULTFILEFORMAT("hive.default.fileformat", VarType.STRING, + new StringsValidator("TextFile")), + HIVEQUERYRESULTFILEFORMAT("hive.query.result.fileformat", VarType.STRING, + new StringsValidator("TextFile")), + HIVECHECKFILEFORMAT("hive.fileformat.check", VarType.BOOLEAN), // default serde for rcfile - HIVEDEFAULTRCFILESERDE("hive.default.rcfile.serde", - "org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe"), + HIVEDEFAULTRCFILESERDE("hive.default.rcfile.serde"), //Location of Hive run time structured log file - HIVEHISTORYFILELOC("hive.querylog.location", System.getProperty("java.io.tmpdir") + File.separator + System.getProperty("user.name")), + HIVEHISTORYFILELOC("hive.querylog.location"), // Whether to log the plan's progress every time a job's progress is checked - HIVE_LOG_INCREMENTAL_PLAN_PROGRESS("hive.querylog.enable.plan.progress", true), + HIVE_LOG_INCREMENTAL_PLAN_PROGRESS("hive.querylog.enable.plan.progress", VarType.BOOLEAN), // The interval between logging the plan's progress in milliseconds - HIVE_LOG_INCREMENTAL_PLAN_PROGRESS_INTERVAL("hive.querylog.plan.progress.interval", 60000L), + HIVE_LOG_INCREMENTAL_PLAN_PROGRESS_INTERVAL("hive.querylog.plan.progress.interval", VarType.LONG), // Default serde and record reader for user scripts - HIVESCRIPTSERDE("hive.script.serde", "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"), - HIVESCRIPTRECORDREADER("hive.script.recordreader", - "org.apache.hadoop.hive.ql.exec.TextRecordReader"), - HIVESCRIPTRECORDWRITER("hive.script.recordwriter", - "org.apache.hadoop.hive.ql.exec.TextRecordWriter"), - HIVESCRIPTESCAPE("hive.transform.escape.input", false), - HIVEBINARYRECORDMAX("hive.binary.record.max.length", 1000 ), + HIVESCRIPTSERDE("hive.script.serde"), + HIVESCRIPTRECORDREADER("hive.script.recordreader"), + HIVESCRIPTRECORDWRITER("hive.script.recordwriter"), + HIVESCRIPTESCAPE("hive.transform.escape.input", VarType.BOOLEAN), + HIVEBINARYRECORDMAX("hive.binary.record.max.length", VarType.INT), // HWI - HIVEHWILISTENHOST("hive.hwi.listen.host", "0.0.0.0"), - HIVEHWILISTENPORT("hive.hwi.listen.port", "9999"), - HIVEHWIWARFILE("hive.hwi.war.file", System.getenv("HWI_WAR_FILE")), + HIVEHWILISTENHOST("hive.hwi.listen.host"), + HIVEHWILISTENPORT("hive.hwi.listen.port"), + HIVEHWIWARFILE("hive.hwi.war.file"), // mapper/reducer memory in local mode - HIVEHADOOPMAXMEM("hive.mapred.local.mem", 0), + HIVEHADOOPMAXMEM("hive.mapred.local.mem", VarType.INT), //small table file size - HIVESMALLTABLESFILESIZE("hive.mapjoin.smalltable.filesize",25000000L), //25M + HIVESMALLTABLESFILESIZE("hive.mapjoin.smalltable.filesize", VarType.LONG), //25M // random number for split sampling - HIVESAMPLERANDOMNUM("hive.sample.seednumber", 0), + HIVESAMPLERANDOMNUM("hive.sample.seednumber", VarType.INT), // test mode in hive mode - HIVETESTMODE("hive.test.mode", false), - HIVETESTMODEPREFIX("hive.test.mode.prefix", "test_"), - HIVETESTMODESAMPLEFREQ("hive.test.mode.samplefreq", 32), - HIVETESTMODENOSAMPLE("hive.test.mode.nosamplelist", ""), - - HIVEMERGEMAPFILES("hive.merge.mapfiles", true), - HIVEMERGEMAPREDFILES("hive.merge.mapredfiles", false), - HIVEMERGEMAPFILESSIZE("hive.merge.size.per.task", (long) (256 * 1000 * 1000)), - HIVEMERGEMAPFILESAVGSIZE("hive.merge.smallfiles.avgsize", (long) (16 * 1000 * 1000)), - HIVEMERGERCFILEBLOCKLEVEL("hive.merge.rcfile.block.level", true), - HIVEMERGEINPUTFORMATBLOCKLEVEL("hive.merge.input.format.block.level", - "org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileBlockMergeInputFormat"), + HIVETESTMODE("hive.test.mode", VarType.BOOLEAN), + HIVETESTMODEPREFIX("hive.test.mode.prefix"), + HIVETESTMODESAMPLEFREQ("hive.test.mode.samplefreq", VarType.INT), + HIVETESTMODENOSAMPLE("hive.test.mode.nosamplelist", true), + + HIVEMERGEMAPFILES("hive.merge.mapfiles", VarType.BOOLEAN), + HIVEMERGEMAPREDFILES("hive.merge.mapredfiles", VarType.BOOLEAN), + HIVEMERGEMAPFILESSIZE("hive.merge.size.per.task", VarType.LONG), + HIVEMERGEMAPFILESAVGSIZE("hive.merge.smallfiles.avgsize", VarType.LONG), + HIVEMERGERCFILEBLOCKLEVEL("hive.merge.rcfile.block.level", VarType.BOOLEAN), + HIVEMERGEINPUTFORMATBLOCKLEVEL("hive.merge.input.format.block.level"), HIVEMERGECURRENTJOBHASDYNAMICPARTITIONS( - "hive.merge.current.job.has.dynamic.partitions", false), + "hive.merge.current.job.has.dynamic.partitions", VarType.BOOLEAN), - HIVEUSEEXPLICITRCFILEHEADER("hive.exec.rcfile.use.explicit.header", true), - HIVEUSERCFILESYNCCACHE("hive.exec.rcfile.use.sync.cache", true), + HIVEUSEEXPLICITRCFILEHEADER("hive.exec.rcfile.use.explicit.header", VarType.BOOLEAN), + HIVEUSERCFILESYNCCACHE("hive.exec.rcfile.use.sync.cache", VarType.BOOLEAN), // Maximum fraction of heap that can be used by ORC file writers - HIVE_ORC_FILE_MEMORY_POOL("hive.exec.orc.memory.pool", 0.5f), // 50% + HIVE_ORC_FILE_MEMORY_POOL("hive.exec.orc.memory.pool", VarType.FLOAT), // 50% // Define the version of the file to write - HIVE_ORC_WRITE_FORMAT("hive.exec.orc.write.format", null), + HIVE_ORC_WRITE_FORMAT("hive.exec.orc.write.format"), // Define the default ORC stripe size HIVE_ORC_DEFAULT_STRIPE_SIZE("hive.exec.orc.default.stripe.size", - 256L * 1024 * 1024), - - HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD("hive.exec.orc.dictionary.key.size.threshold", 0.8f), - - HIVESKEWJOIN("hive.optimize.skewjoin", false), - HIVECONVERTJOIN("hive.auto.convert.join", true), - HIVECONVERTJOINNOCONDITIONALTASK("hive.auto.convert.join.noconditionaltask", true), - HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD("hive.auto.convert.join.noconditionaltask.size", - 10000000L), - HIVESKEWJOINKEY("hive.skewjoin.key", 100000), - HIVESKEWJOINMAPJOINNUMMAPTASK("hive.skewjoin.mapjoin.map.tasks", 10000), - HIVESKEWJOINMAPJOINMINSPLIT("hive.skewjoin.mapjoin.min.split", 33554432L), //32M - - HIVESENDHEARTBEAT("hive.heartbeat.interval", 1000), - HIVELIMITMAXROWSIZE("hive.limit.row.max.size", 100000L), - HIVELIMITOPTLIMITFILE("hive.limit.optimize.limit.file", 10), - HIVELIMITOPTENABLE("hive.limit.optimize.enable", false), - HIVELIMITOPTMAXFETCH("hive.limit.optimize.fetch.max", 50000), - HIVELIMITPUSHDOWNMEMORYUSAGE("hive.limit.pushdown.memory.usage", -1f), - - HIVEHASHTABLETHRESHOLD("hive.hashtable.initialCapacity", 100000), - HIVEHASHTABLELOADFACTOR("hive.hashtable.loadfactor", (float) 0.75), - HIVEHASHTABLEFOLLOWBYGBYMAXMEMORYUSAGE("hive.mapjoin.followby.gby.localtask.max.memory.usage", (float) 0.55), - HIVEHASHTABLEMAXMEMORYUSAGE("hive.mapjoin.localtask.max.memory.usage", (float) 0.90), - HIVEHASHTABLESCALE("hive.mapjoin.check.memory.rows", (long)100000), - - HIVEDEBUGLOCALTASK("hive.debug.localtask",false), - - HIVEINPUTFORMAT("hive.input.format", "org.apache.hadoop.hive.ql.io.CombineHiveInputFormat"), - - HIVEENFORCEBUCKETING("hive.enforce.bucketing", false), - HIVEENFORCESORTING("hive.enforce.sorting", false), - HIVEOPTIMIZEBUCKETINGSORTING("hive.optimize.bucketingsorting", true), - HIVEPARTITIONER("hive.mapred.partitioner", "org.apache.hadoop.hive.ql.io.DefaultHivePartitioner"), - HIVEENFORCESORTMERGEBUCKETMAPJOIN("hive.enforce.sortmergebucketmapjoin", false), - HIVEENFORCEBUCKETMAPJOIN("hive.enforce.bucketmapjoin", false), - - HIVE_AUTO_SORTMERGE_JOIN("hive.auto.convert.sortmerge.join", false), + VarType.INT), + + HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD("hive.exec.orc.dictionary.key.size.threshold", VarType.FLOAT), + + HIVESKEWJOIN("hive.optimize.skewjoin", VarType.BOOLEAN), + HIVECONVERTJOIN("hive.auto.convert.join", VarType.BOOLEAN), + HIVECONVERTJOINNOCONDITIONALTASK("hive.auto.convert.join.noconditionaltask", VarType.BOOLEAN), + HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD("hive.auto.convert.join.noconditionaltask.size", VarType.LONG), + HIVESKEWJOINKEY("hive.skewjoin.key", VarType.INT), + HIVESKEWJOINMAPJOINNUMMAPTASK("hive.skewjoin.mapjoin.map.tasks", VarType.INT), + HIVESKEWJOINMAPJOINMINSPLIT("hive.skewjoin.mapjoin.min.split", VarType.LONG), //32M + + HIVESENDHEARTBEAT("hive.heartbeat.interval", VarType.INT), + HIVELIMITMAXROWSIZE("hive.limit.row.max.size", VarType.LONG), + HIVELIMITOPTLIMITFILE("hive.limit.optimize.limit.file", VarType.INT), + HIVELIMITOPTENABLE("hive.limit.optimize.enable", VarType.BOOLEAN), + HIVELIMITOPTMAXFETCH("hive.limit.optimize.fetch.max", VarType.INT), + HIVELIMITPUSHDOWNMEMORYUSAGE("hive.limit.pushdown.memory.usage", VarType.FLOAT), + + HIVEHASHTABLETHRESHOLD("hive.hashtable.initialCapacity", VarType.INT), + HIVEHASHTABLELOADFACTOR("hive.hashtable.loadfactor", VarType.FLOAT), + HIVEHASHTABLEFOLLOWBYGBYMAXMEMORYUSAGE("hive.mapjoin.followby.gby.localtask.max.memory.usage", VarType.FLOAT), + HIVEHASHTABLEMAXMEMORYUSAGE("hive.mapjoin.localtask.max.memory.usage", VarType.FLOAT), + HIVEHASHTABLESCALE("hive.mapjoin.check.memory.rows", VarType.LONG), + + HIVEDEBUGLOCALTASK("hive.debug.localtask", VarType.BOOLEAN), + + HIVEINPUTFORMAT("hive.input.format"), + + HIVEENFORCEBUCKETING("hive.enforce.bucketing", VarType.BOOLEAN), + HIVEENFORCESORTING("hive.enforce.sorting", VarType.BOOLEAN), + HIVEOPTIMIZEBUCKETINGSORTING("hive.optimize.bucketingsorting", VarType.BOOLEAN), + HIVEPARTITIONER("hive.mapred.partitioner"), + HIVEENFORCESORTMERGEBUCKETMAPJOIN("hive.enforce.sortmergebucketmapjoin", VarType.BOOLEAN), + HIVEENFORCEBUCKETMAPJOIN("hive.enforce.bucketmapjoin", VarType.BOOLEAN), + + HIVE_AUTO_SORTMERGE_JOIN("hive.auto.convert.sortmerge.join", VarType.BOOLEAN), HIVE_AUTO_SORTMERGE_JOIN_BIGTABLE_SELECTOR( - "hive.auto.convert.sortmerge.join.bigtable.selection.policy", - "org.apache.hadoop.hive.ql.optimizer.AvgPartitionSizeBasedBigTableSelectorForAutoSMJ"), + "hive.auto.convert.sortmerge.join.bigtable.selection.policy"), HIVE_AUTO_SORTMERGE_JOIN_TOMAPJOIN( - "hive.auto.convert.sortmerge.join.to.mapjoin", false), + "hive.auto.convert.sortmerge.join.to.mapjoin", VarType.BOOLEAN), - HIVESCRIPTOPERATORTRUST("hive.exec.script.trust", false), - HIVEROWOFFSET("hive.exec.rowoffset", false), + HIVESCRIPTOPERATORTRUST("hive.exec.script.trust", VarType.BOOLEAN), + HIVEROWOFFSET("hive.exec.rowoffset", VarType.BOOLEAN), - HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE("hive.hadoop.supports.splittable.combineinputformat", false), + HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE("hive.hadoop.supports.splittable.combineinputformat", VarType.BOOLEAN), // Optimizer - HIVEOPTINDEXFILTER("hive.optimize.index.filter", false), // automatically use indexes - HIVEINDEXAUTOUPDATE("hive.optimize.index.autoupdate", false), //automatically update stale indexes - HIVEOPTPPD("hive.optimize.ppd", true), // predicate pushdown - HIVEPPDRECOGNIZETRANSITIVITY("hive.ppd.recognizetransivity", true), // predicate pushdown - HIVEPPDREMOVEDUPLICATEFILTERS("hive.ppd.remove.duplicatefilters", true), - HIVEMETADATAONLYQUERIES("hive.optimize.metadataonly", true), + HIVEOPTINDEXFILTER("hive.optimize.index.filter", VarType.BOOLEAN), // automatically use indexes + HIVEINDEXAUTOUPDATE("hive.optimize.index.autoupdate", VarType.BOOLEAN), //automatically update stale indexes + HIVEOPTPPD("hive.optimize.ppd", VarType.BOOLEAN), // predicate pushdown + HIVEPPDRECOGNIZETRANSITIVITY("hive.ppd.recognizetransivity", VarType.BOOLEAN), // predicate pushdown + HIVEPPDREMOVEDUPLICATEFILTERS("hive.ppd.remove.duplicatefilters", VarType.BOOLEAN), + HIVEMETADATAONLYQUERIES("hive.optimize.metadataonly", VarType.BOOLEAN), // push predicates down to storage handlers - HIVEOPTPPD_STORAGE("hive.optimize.ppd.storage", true), - HIVEOPTGROUPBY("hive.optimize.groupby", true), // optimize group by - HIVEOPTBUCKETMAPJOIN("hive.optimize.bucketmapjoin", false), // optimize bucket map join - HIVEOPTSORTMERGEBUCKETMAPJOIN("hive.optimize.bucketmapjoin.sortedmerge", false), // try to use sorted merge bucket map join - HIVEOPTREDUCEDEDUPLICATION("hive.optimize.reducededuplication", true), - HIVEOPTREDUCEDEDUPLICATIONMINREDUCER("hive.optimize.reducededuplication.min.reducer", 4), + HIVEOPTPPD_STORAGE("hive.optimize.ppd.storage", VarType.BOOLEAN), + HIVEOPTGROUPBY("hive.optimize.groupby", VarType.BOOLEAN), // optimize group by + HIVEOPTBUCKETMAPJOIN("hive.optimize.bucketmapjoin", VarType.BOOLEAN), // optimize bucket map join + HIVEOPTSORTMERGEBUCKETMAPJOIN("hive.optimize.bucketmapjoin.sortedmerge", VarType.BOOLEAN), // try to use sorted merge bucket map join + HIVEOPTREDUCEDEDUPLICATION("hive.optimize.reducededuplication", VarType.BOOLEAN), + HIVEOPTREDUCEDEDUPLICATIONMINREDUCER("hive.optimize.reducededuplication.min.reducer", VarType.INT), - HIVESAMPLINGFORORDERBY("hive.optimize.sampling.orderby", false), - HIVESAMPLINGNUMBERFORORDERBY("hive.optimize.sampling.orderby.number", 1000), - HIVESAMPLINGPERCENTFORORDERBY("hive.optimize.sampling.orderby.percent", 0.1f), + HIVESAMPLINGFORORDERBY("hive.optimize.sampling.orderby", VarType.BOOLEAN), + HIVESAMPLINGNUMBERFORORDERBY("hive.optimize.sampling.orderby.number", VarType.INT), + HIVESAMPLINGPERCENTFORORDERBY("hive.optimize.sampling.orderby.percent", VarType.FLOAT), // whether to optimize union followed by select followed by filesink // It creates sub-directories in the final output, so should not be turned on in systems // where MAPREDUCE-1501 is not present - HIVE_OPTIMIZE_UNION_REMOVE("hive.optimize.union.remove", false), - HIVEOPTCORRELATION("hive.optimize.correlation", false), // exploit intra-query correlations + HIVE_OPTIMIZE_UNION_REMOVE("hive.optimize.union.remove", VarType.BOOLEAN), + HIVEOPTCORRELATION("hive.optimize.correlation", VarType.BOOLEAN), // exploit intra-query correlations // whether hadoop map-reduce supports sub-directories. It was added by MAPREDUCE-1501. // Some optimizations can only be performed if the version of hadoop being used supports // sub-directories - HIVE_HADOOP_SUPPORTS_SUBDIRECTORIES("hive.mapred.supports.subdirectories", false), + HIVE_HADOOP_SUPPORTS_SUBDIRECTORIES("hive.mapred.supports.subdirectories", VarType.BOOLEAN), // optimize skewed join by changing the query plan at compile time - HIVE_OPTIMIZE_SKEWJOIN_COMPILETIME("hive.optimize.skewjoin.compiletime", false), + HIVE_OPTIMIZE_SKEWJOIN_COMPILETIME("hive.optimize.skewjoin.compiletime", VarType.BOOLEAN), // Indexes - HIVEOPTINDEXFILTER_COMPACT_MINSIZE("hive.optimize.index.filter.compact.minsize", (long) 5 * 1024 * 1024 * 1024), // 5G - HIVEOPTINDEXFILTER_COMPACT_MAXSIZE("hive.optimize.index.filter.compact.maxsize", (long) -1), // infinity - HIVE_INDEX_COMPACT_QUERY_MAX_ENTRIES("hive.index.compact.query.max.entries", (long) 10000000), // 10M - HIVE_INDEX_COMPACT_QUERY_MAX_SIZE("hive.index.compact.query.max.size", (long) 10 * 1024 * 1024 * 1024), // 10G - HIVE_INDEX_COMPACT_BINARY_SEARCH("hive.index.compact.binary.search", true), + HIVEOPTINDEXFILTER_COMPACT_MINSIZE("hive.optimize.index.filter.compact.minsize", VarType.LONG), // 5G + HIVEOPTINDEXFILTER_COMPACT_MAXSIZE("hive.optimize.index.filter.compact.maxsize", VarType.LONG), // infinity + HIVE_INDEX_COMPACT_QUERY_MAX_ENTRIES("hive.index.compact.query.max.entries", VarType.LONG), // 10M + HIVE_INDEX_COMPACT_QUERY_MAX_SIZE("hive.index.compact.query.max.size", VarType.LONG), // 10G + HIVE_INDEX_COMPACT_BINARY_SEARCH("hive.index.compact.binary.search", VarType.BOOLEAN), // Statistics - HIVESTATSAUTOGATHER("hive.stats.autogather", true), - HIVESTATSDBCLASS("hive.stats.dbclass", "counter", - new PatternValidator("jdbc(:.*)", "hbase", "counter", "custom")), // StatsSetupConst.StatDB - HIVESTATSJDBCDRIVER("hive.stats.jdbcdriver", - "org.apache.derby.jdbc.EmbeddedDriver"), // JDBC driver specific to the dbclass - HIVESTATSDBCONNECTIONSTRING("hive.stats.dbconnectionstring", - "jdbc:derby:;databaseName=TempStatsStore;create=true"), // automatically create database - HIVE_STATS_DEFAULT_PUBLISHER("hive.stats.default.publisher", - ""), // default stats publisher if none of JDBC/HBase is specified - HIVE_STATS_DEFAULT_AGGREGATOR("hive.stats.default.aggregator", - ""), // default stats aggregator if none of JDBC/HBase is specified - HIVE_STATS_JDBC_TIMEOUT("hive.stats.jdbc.timeout", - 30), // default timeout in sec for JDBC connection & SQL statements - HIVE_STATS_ATOMIC("hive.stats.atomic", - false), // whether to update metastore stats only if all stats are available - HIVE_STATS_RETRIES_MAX("hive.stats.retries.max", - 0), // maximum # of retries to insert/select/delete the stats DB - HIVE_STATS_RETRIES_WAIT("hive.stats.retries.wait", - 3000), // # milliseconds to wait before the next retry - HIVE_STATS_COLLECT_RAWDATASIZE("hive.stats.collect.rawdatasize", true), + HIVESTATSAUTOGATHER("hive.stats.autogather", VarType.BOOLEAN), + HIVESTATSDBCLASS("hive.stats.dbclass", VarType.STRING, + new PatternValidator("jdbc(:.*)")), // StatsSetupConst.StatDB + HIVESTATSJDBCDRIVER("hive.stats.jdbcdriver"), // JDBC driver specific to the dbclass + HIVESTATSDBCONNECTIONSTRING("hive.stats.dbconnectionstring"), // automatically create database + HIVE_STATS_DEFAULT_PUBLISHER("hive.stats.default.publisher", true), // default stats publisher if none of JDBC/HBase is specified + HIVE_STATS_DEFAULT_AGGREGATOR("hive.stats.default.aggregator", true), // default stats aggregator if none of JDBC/HBase is specified + HIVE_STATS_JDBC_TIMEOUT("hive.stats.jdbc.timeout", VarType.INT), // default timeout in sec for JDBC connection & SQL statements + HIVE_STATS_ATOMIC("hive.stats.atomic", VarType.BOOLEAN), // whether to update metastore stats only if all stats are available + HIVE_STATS_RETRIES_MAX("hive.stats.retries.max", VarType.INT), // maximum # of retries to insert/select/delete the stats DB + HIVE_STATS_RETRIES_WAIT("hive.stats.retries.wait", VarType.INT), // # milliseconds to wait before the next retry + HIVE_STATS_COLLECT_RAWDATASIZE("hive.stats.collect.rawdatasize", VarType.BOOLEAN), // should the raw data size be collected when analyzing tables - CLIENT_STATS_COUNTERS("hive.client.stats.counters", ""), + CLIENT_STATS_COUNTERS("hive.client.stats.counters", true), //Subset of counters that should be of interest for hive.client.stats.publishers (when one wants to limit their publishing). Non-display names should be used". - HIVE_STATS_RELIABLE("hive.stats.reliable", false), + HIVE_STATS_RELIABLE("hive.stats.reliable", VarType.BOOLEAN), // Collect table access keys information for operators that can benefit from bucketing - HIVE_STATS_COLLECT_TABLEKEYS("hive.stats.collect.tablekeys", false), + HIVE_STATS_COLLECT_TABLEKEYS("hive.stats.collect.tablekeys", VarType.BOOLEAN), // Collect column access information - HIVE_STATS_COLLECT_SCANCOLS("hive.stats.collect.scancols", false), + HIVE_STATS_COLLECT_SCANCOLS("hive.stats.collect.scancols", VarType.BOOLEAN), // standard error allowed for ndv estimates. A lower value indicates higher accuracy and a // higher compute cost. - HIVE_STATS_NDV_ERROR("hive.stats.ndv.error", (float)20.0), - HIVE_STATS_KEY_PREFIX_MAX_LENGTH("hive.stats.key.prefix.max.length", 150), - HIVE_STATS_KEY_PREFIX("hive.stats.key.prefix", ""), // internal usage only + HIVE_STATS_NDV_ERROR("hive.stats.ndv.error", VarType.FLOAT), + HIVE_STATS_KEY_PREFIX_MAX_LENGTH("hive.stats.key.prefix.max.length", VarType.INT), + HIVE_STATS_KEY_PREFIX("hive.stats.key.prefix", true), // internal usage only // if length of variable length data type cannot be determined this length will be used. - HIVE_STATS_MAX_VARIABLE_LENGTH("hive.stats.max.variable.length", 100), + HIVE_STATS_MAX_VARIABLE_LENGTH("hive.stats.max.variable.length", VarType.INT), // if number of elements in list cannot be determined, this value will be used - HIVE_STATS_LIST_NUM_ENTRIES("hive.stats.list.num.entries", 10), + HIVE_STATS_LIST_NUM_ENTRIES("hive.stats.list.num.entries", VarType.INT), // if number of elements in map cannot be determined, this value will be used - HIVE_STATS_MAP_NUM_ENTRIES("hive.stats.map.num.entries", 10), + HIVE_STATS_MAP_NUM_ENTRIES("hive.stats.map.num.entries", VarType.INT), // to accurately compute statistics for GROUPBY map side parallelism needs to be known - HIVE_STATS_MAP_SIDE_PARALLELISM("hive.stats.map.parallelism", 1), + HIVE_STATS_MAP_SIDE_PARALLELISM("hive.stats.map.parallelism", VarType.INT), // statistics annotation fetches column statistics for all required columns and for all // required partitions which can be very expensive sometimes - HIVE_STATS_FETCH_COLUMN_STATS("hive.stats.fetch.column.stats", false), + HIVE_STATS_FETCH_COLUMN_STATS("hive.stats.fetch.column.stats", VarType.BOOLEAN), // in the absence of table/partition stats, average row size will be used to // estimate the number of rows/data size - HIVE_STATS_AVG_ROW_SIZE("hive.stats.avg.row.size", 10000), + HIVE_STATS_AVG_ROW_SIZE("hive.stats.avg.row.size", VarType.INT), // in the absence of column statistics, the estimated number of rows/data size that will // emitted from join operator will depend on t factor - HIVE_STATS_JOIN_FACTOR("hive.stats.join.factor", (float) 1.1), + HIVE_STATS_JOIN_FACTOR("hive.stats.join.factor", VarType.FLOAT), // in the absence of uncompressed/raw data size, total file size will be used for statistics // annotation. But the file may be compressed, encoded and serialized which may be lesser in size // than the actual uncompressed/raw data size. This factor will be multiplied to file size to estimate // the raw data size. - HIVE_STATS_DESERIALIZATION_FACTOR("hive.stats.deserialization.factor", (float) 1.0), + HIVE_STATS_DESERIALIZATION_FACTOR("hive.stats.deserialization.factor", VarType.FLOAT), // Concurrency - HIVE_SUPPORT_CONCURRENCY("hive.support.concurrency", false), - HIVE_LOCK_MANAGER("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager"), - HIVE_LOCK_NUMRETRIES("hive.lock.numretries", 100), - HIVE_UNLOCK_NUMRETRIES("hive.unlock.numretries", 10), - HIVE_LOCK_SLEEP_BETWEEN_RETRIES("hive.lock.sleep.between.retries", 60), - HIVE_LOCK_MAPRED_ONLY("hive.lock.mapred.only.operation", false), - - HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", ""), - HIVE_ZOOKEEPER_CLIENT_PORT("hive.zookeeper.client.port", "2181"), - HIVE_ZOOKEEPER_SESSION_TIMEOUT("hive.zookeeper.session.timeout", 600*1000), - HIVE_ZOOKEEPER_NAMESPACE("hive.zookeeper.namespace", "hive_zookeeper_namespace"), - HIVE_ZOOKEEPER_CLEAN_EXTRA_NODES("hive.zookeeper.clean.extra.nodes", false), + HIVE_SUPPORT_CONCURRENCY("hive.support.concurrency", VarType.BOOLEAN), + HIVE_LOCK_MANAGER("hive.lock.manager"), + HIVE_LOCK_NUMRETRIES("hive.lock.numretries", VarType.INT), + HIVE_UNLOCK_NUMRETRIES("hive.unlock.numretries", VarType.INT), + HIVE_LOCK_SLEEP_BETWEEN_RETRIES("hive.lock.sleep.between.retries", VarType.INT), + HIVE_LOCK_MAPRED_ONLY("hive.lock.mapred.only.operation", VarType.BOOLEAN), + + HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", true), + HIVE_ZOOKEEPER_CLIENT_PORT("hive.zookeeper.client.port"), + HIVE_ZOOKEEPER_SESSION_TIMEOUT("hive.zookeeper.session.timeout", VarType.INT), + HIVE_ZOOKEEPER_NAMESPACE("hive.zookeeper.namespace"), + HIVE_ZOOKEEPER_CLEAN_EXTRA_NODES("hive.zookeeper.clean.extra.nodes", VarType.BOOLEAN), // For HBase storage handler - HIVE_HBASE_WAL_ENABLED("hive.hbase.wal.enabled", true), + HIVE_HBASE_WAL_ENABLED("hive.hbase.wal.enabled", VarType.BOOLEAN), // For har files - HIVEARCHIVEENABLED("hive.archive.enabled", false), + HIVEARCHIVEENABLED("hive.archive.enabled", VarType.BOOLEAN), //Enable/Disable gbToIdx rewrite rule - HIVEOPTGBYUSINGINDEX("hive.optimize.index.groupby", false), + HIVEOPTGBYUSINGINDEX("hive.optimize.index.groupby", VarType.BOOLEAN), - HIVEOUTERJOINSUPPORTSFILTERS("hive.outerjoin.supports.filters", true), + HIVEOUTERJOINSUPPORTSFILTERS("hive.outerjoin.supports.filters", VarType.BOOLEAN), // 'minimal', 'more' (and 'all' later) - HIVEFETCHTASKCONVERSION("hive.fetch.task.conversion", "minimal", - new StringsValidator("minimal", "more")), - HIVEFETCHTASKCONVERSIONTHRESHOLD("hive.fetch.task.conversion.threshold", -1l), + HIVEFETCHTASKCONVERSION("hive.fetch.task.conversion", VarType.STRING, + new StringsValidator("minimal")), + HIVEFETCHTASKCONVERSIONTHRESHOLD("hive.fetch.task.conversion.threshold", VarType.LONG), - HIVEFETCHTASKAGGR("hive.fetch.task.aggr", false), + HIVEFETCHTASKAGGR("hive.fetch.task.aggr", VarType.BOOLEAN), - HIVEOPTIMIZEMETADATAQUERIES("hive.compute.query.using.stats", false), + HIVEOPTIMIZEMETADATAQUERIES("hive.compute.query.using.stats", VarType.BOOLEAN), // Serde for FetchTask - HIVEFETCHOUTPUTSERDE("hive.fetch.output.serde", "org.apache.hadoop.hive.serde2.DelimitedJSONSerDe"), + HIVEFETCHOUTPUTSERDE("hive.fetch.output.serde"), - HIVEEXPREVALUATIONCACHE("hive.cache.expr.evaluation", true), + HIVEEXPREVALUATIONCACHE("hive.cache.expr.evaluation", VarType.BOOLEAN), // Hive Variables - HIVEVARIABLESUBSTITUTE("hive.variable.substitute", true), - HIVEVARIABLESUBSTITUTEDEPTH("hive.variable.substitute.depth", 40), - - HIVECONFVALIDATION("hive.conf.validation", true), - - SEMANTIC_ANALYZER_HOOK("hive.semantic.analyzer.hook", ""), - - HIVE_AUTHORIZATION_ENABLED("hive.security.authorization.enabled", false), - HIVE_AUTHORIZATION_MANAGER("hive.security.authorization.manager", - "org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider"), - HIVE_AUTHENTICATOR_MANAGER("hive.security.authenticator.manager", - "org.apache.hadoop.hive.ql.security.HadoopDefaultAuthenticator"), - HIVE_METASTORE_AUTHORIZATION_MANAGER("hive.security.metastore.authorization.manager", - "org.apache.hadoop.hive.ql.security.authorization." - + "DefaultHiveMetastoreAuthorizationProvider"), - HIVE_METASTORE_AUTHENTICATOR_MANAGER("hive.security.metastore.authenticator.manager", - "org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator"), - HIVE_AUTHORIZATION_TABLE_USER_GRANTS("hive.security.authorization.createtable.user.grants", ""), - HIVE_AUTHORIZATION_TABLE_GROUP_GRANTS("hive.security.authorization.createtable.group.grants", - ""), - HIVE_AUTHORIZATION_TABLE_ROLE_GRANTS("hive.security.authorization.createtable.role.grants", ""), - HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS("hive.security.authorization.createtable.owner.grants", - ""), + HIVEVARIABLESUBSTITUTE("hive.variable.substitute", VarType.BOOLEAN), + HIVEVARIABLESUBSTITUTEDEPTH("hive.variable.substitute.depth", VarType.INT), + + HIVECONFVALIDATION("hive.conf.validation", VarType.BOOLEAN), + + SEMANTIC_ANALYZER_HOOK("hive.semantic.analyzer.hook", true), + + HIVE_AUTHORIZATION_ENABLED("hive.security.authorization.enabled", VarType.BOOLEAN), + HIVE_AUTHORIZATION_MANAGER("hive.security.authorization.manager"), + HIVE_AUTHENTICATOR_MANAGER("hive.security.authenticator.manager"), + HIVE_METASTORE_AUTHORIZATION_MANAGER("hive.security.metastore.authorization.manager"), + HIVE_METASTORE_AUTHENTICATOR_MANAGER("hive.security.metastore.authenticator.manager"), + HIVE_AUTHORIZATION_TABLE_USER_GRANTS("hive.security.authorization.createtable.user.grants", true), + HIVE_AUTHORIZATION_TABLE_GROUP_GRANTS("hive.security.authorization.createtable.group.grants", true), + HIVE_AUTHORIZATION_TABLE_ROLE_GRANTS("hive.security.authorization.createtable.role.grants", true), + HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS("hive.security.authorization.createtable.owner.grants", true), // Print column names in output - HIVE_CLI_PRINT_HEADER("hive.cli.print.header", false), + HIVE_CLI_PRINT_HEADER("hive.cli.print.header", VarType.BOOLEAN), - HIVE_ERROR_ON_EMPTY_PARTITION("hive.error.on.empty.partition", false), + HIVE_ERROR_ON_EMPTY_PARTITION("hive.error.on.empty.partition", VarType.BOOLEAN), - HIVE_INDEX_IGNORE_HDFS_LOC("hive.index.compact.file.ignore.hdfs", false), + HIVE_INDEX_IGNORE_HDFS_LOC("hive.index.compact.file.ignore.hdfs", VarType.BOOLEAN), - HIVE_EXIM_URI_SCHEME_WL("hive.exim.uri.scheme.whitelist", "hdfs,pfile"), + HIVE_EXIM_URI_SCHEME_WL("hive.exim.uri.scheme.whitelist"), // temporary variable for testing. This is added just to turn off this feature in case of a bug in // deployment. It has not been documented in hive-default.xml intentionally, this should be removed // once the feature is stable - HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS("hive.mapper.cannot.span.multiple.partitions", false), - HIVE_REWORK_MAPREDWORK("hive.rework.mapredwork", false), - HIVE_CONCATENATE_CHECK_INDEX ("hive.exec.concatenate.check.index", true), - HIVE_IO_EXCEPTION_HANDLERS("hive.io.exception.handlers", ""), + HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS("hive.mapper.cannot.span.multiple.partitions", VarType.BOOLEAN), + HIVE_REWORK_MAPREDWORK("hive.rework.mapredwork", VarType.BOOLEAN), + HIVE_CONCATENATE_CHECK_INDEX ("hive.exec.concatenate.check.index", VarType.BOOLEAN), + HIVE_IO_EXCEPTION_HANDLERS("hive.io.exception.handlers", true), // logging configuration - HIVE_LOG4J_FILE("hive.log4j.file", ""), - HIVE_EXEC_LOG4J_FILE("hive.exec.log4j.file", ""), + HIVE_LOG4J_FILE("hive.log4j.file", true), + HIVE_EXEC_LOG4J_FILE("hive.exec.log4j.file", true), // prefix used to auto generated column aliases (this should be started with '_') - HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL("hive.autogen.columnalias.prefix.label", "_c"), + HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL("hive.autogen.columnalias.prefix.label"), HIVE_AUTOGEN_COLUMNALIAS_PREFIX_INCLUDEFUNCNAME( - "hive.autogen.columnalias.prefix.includefuncname", false), + "hive.autogen.columnalias.prefix.includefuncname", VarType.BOOLEAN), // The class responsible for logging client side performance metrics // Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger - HIVE_PERF_LOGGER("hive.exec.perf.logger", "org.apache.hadoop.hive.ql.log.PerfLogger"), + HIVE_PERF_LOGGER("hive.exec.perf.logger"), // Whether to delete the scratchdir while startup - HIVE_START_CLEANUP_SCRATCHDIR("hive.start.cleanup.scratchdir", false), - HIVE_INSERT_INTO_MULTILEVEL_DIRS("hive.insert.into.multilevel.dirs", false), - HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS("hive.warehouse.subdir.inherit.perms", false), + HIVE_START_CLEANUP_SCRATCHDIR("hive.start.cleanup.scratchdir", VarType.BOOLEAN), + HIVE_INSERT_INTO_MULTILEVEL_DIRS("hive.insert.into.multilevel.dirs", VarType.BOOLEAN), + HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS("hive.warehouse.subdir.inherit.perms", VarType.BOOLEAN), // whether insert into external tables is allowed - HIVE_INSERT_INTO_EXTERNAL_TABLES("hive.insert.into.external.tables", true), + HIVE_INSERT_INTO_EXTERNAL_TABLES("hive.insert.into.external.tables", VarType.BOOLEAN), // A comma separated list of hooks which implement HiveDriverRunHook and will be run at the // beginning and end of Driver.run, these will be run in the order specified - HIVE_DRIVER_RUN_HOOKS("hive.exec.driver.run.hooks", ""), - HIVE_DDL_OUTPUT_FORMAT("hive.ddl.output.format", null), - HIVE_ENTITY_SEPARATOR("hive.entity.separator", "@"), + HIVE_DRIVER_RUN_HOOKS("hive.exec.driver.run.hooks", true), + HIVE_DDL_OUTPUT_FORMAT("hive.ddl.output.format"), + HIVE_ENTITY_SEPARATOR("hive.entity.separator"), // binary or http - HIVE_SERVER2_TRANSPORT_MODE("hive.server2.transport.mode", "binary", - new StringsValidator("binary", "http")), + HIVE_SERVER2_TRANSPORT_MODE("hive.server2.transport.mode", VarType.STRING, + new StringsValidator("binary")), // http (over thrift) transport settings - HIVE_SERVER2_THRIFT_HTTP_PORT("hive.server2.thrift.http.port", 10001), - HIVE_SERVER2_THRIFT_HTTP_PATH("hive.server2.thrift.http.path", "cliservice"), - HIVE_SERVER2_THRIFT_HTTP_MIN_WORKER_THREADS("hive.server2.thrift.http.min.worker.threads", 5), - HIVE_SERVER2_THRIFT_HTTP_MAX_WORKER_THREADS("hive.server2.thrift.http.max.worker.threads", 500), + HIVE_SERVER2_THRIFT_HTTP_PORT("hive.server2.thrift.http.port", VarType.LONG), + HIVE_SERVER2_THRIFT_HTTP_PATH("hive.server2.thrift.http.path", true), + HIVE_SERVER2_THRIFT_HTTP_MIN_WORKER_THREADS("hive.server2.thrift.http.min.worker.threads", VarType.INT), + HIVE_SERVER2_THRIFT_HTTP_MAX_WORKER_THREADS("hive.server2.thrift.http.max.worker.threads", VarType.INT), // binary transport settings - HIVE_SERVER2_THRIFT_PORT("hive.server2.thrift.port", 10000), - HIVE_SERVER2_THRIFT_BIND_HOST("hive.server2.thrift.bind.host", ""), - HIVE_SERVER2_THRIFT_SASL_QOP("hive.server2.thrift.sasl.qop", "auth", - new StringsValidator("auth", "auth-int", "auth-conf")), - HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS("hive.server2.thrift.min.worker.threads", 5), - HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS("hive.server2.thrift.max.worker.threads", 500), + HIVE_SERVER2_THRIFT_PORT("hive.server2.thrift.port", VarType.INT), + HIVE_SERVER2_THRIFT_BIND_HOST("hive.server2.thrift.bind.host"), + HIVE_SERVER2_THRIFT_SASL_QOP("hive.server2.thrift.sasl.qop", VarType.STRING, + new StringsValidator("auth")), + HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS("hive.server2.thrift.min.worker.threads", VarType.INT), + HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS("hive.server2.thrift.max.worker.threads", VarType.INT), // Configuration for async thread pool in SessionManager // Number of async threads - HIVE_SERVER2_ASYNC_EXEC_THREADS("hive.server2.async.exec.threads", 100), + HIVE_SERVER2_ASYNC_EXEC_THREADS("hive.server2.async.exec.threads", VarType.INT), // Number of seconds HiveServer2 shutdown will wait for async threads to terminate - HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT("hive.server2.async.exec.shutdown.timeout", 10L), + HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT("hive.server2.async.exec.shutdown.timeout", VarType.LONG), // Size of the wait queue for async thread pool in HiveServer2. // After hitting this limit, the async thread pool will reject new requests. - HIVE_SERVER2_ASYNC_EXEC_WAIT_QUEUE_SIZE("hive.server2.async.exec.wait.queue.size", 100), + HIVE_SERVER2_ASYNC_EXEC_WAIT_QUEUE_SIZE("hive.server2.async.exec.wait.queue.size", VarType.INT), // Number of seconds that an idle HiveServer2 async thread (from the thread pool) // will wait for a new task to arrive before terminating - HIVE_SERVER2_ASYNC_EXEC_KEEPALIVE_TIME("hive.server2.async.exec.keepalive.time", 10), + HIVE_SERVER2_ASYNC_EXEC_KEEPALIVE_TIME("hive.server2.async.exec.keepalive.time", VarType.INT), // HiveServer2 auth configuration - HIVE_SERVER2_AUTHENTICATION("hive.server2.authentication", "NONE", - new StringsValidator("NOSASL", "NONE", "LDAP", "KERBEROS", "CUSTOM")), - HIVE_SERVER2_KERBEROS_KEYTAB("hive.server2.authentication.kerberos.keytab", ""), - HIVE_SERVER2_KERBEROS_PRINCIPAL("hive.server2.authentication.kerberos.principal", ""), - HIVE_SERVER2_PLAIN_LDAP_URL("hive.server2.authentication.ldap.url", null), - HIVE_SERVER2_PLAIN_LDAP_BASEDN("hive.server2.authentication.ldap.baseDN", null), - HIVE_SERVER2_PLAIN_LDAP_DOMAIN("hive.server2.authentication.ldap.Domain", null), - HIVE_SERVER2_CUSTOM_AUTHENTICATION_CLASS("hive.server2.custom.authentication.class", null), - HIVE_SERVER2_ENABLE_DOAS("hive.server2.enable.doAs", true), - HIVE_SERVER2_TABLE_TYPE_MAPPING("hive.server2.table.type.mapping", "CLASSIC", - new StringsValidator("CLASSIC", "HIVE")), - HIVE_SERVER2_SESSION_HOOK("hive.server2.session.hook", ""), - HIVE_SERVER2_USE_SSL("hive.server2.use.SSL", false), - HIVE_SERVER2_SSL_KEYSTORE_PATH("hive.server2.keystore.path", ""), - HIVE_SERVER2_SSL_KEYSTORE_PASSWORD("hive.server2.keystore.password", ""), - - HIVE_SECURITY_COMMAND_WHITELIST("hive.security.command.whitelist", "set,reset,dfs,add,delete,compile"), - - HIVE_CONF_RESTRICTED_LIST("hive.conf.restricted.list", ""), + HIVE_SERVER2_AUTHENTICATION("hive.server2.authentication", VarType.STRING, + new StringsValidator("NOSASL")), + HIVE_SERVER2_KERBEROS_KEYTAB("hive.server2.authentication.kerberos.keytab", true), + HIVE_SERVER2_KERBEROS_PRINCIPAL("hive.server2.authentication.kerberos.principal", true), + HIVE_SERVER2_PLAIN_LDAP_URL("hive.server2.authentication.ldap.url"), + HIVE_SERVER2_PLAIN_LDAP_BASEDN("hive.server2.authentication.ldap.baseDN"), + HIVE_SERVER2_PLAIN_LDAP_DOMAIN("hive.server2.authentication.ldap.Domain"), + HIVE_SERVER2_CUSTOM_AUTHENTICATION_CLASS("hive.server2.custom.authentication.class"), + HIVE_SERVER2_ENABLE_DOAS("hive.server2.enable.doAs", VarType.BOOLEAN), + HIVE_SERVER2_TABLE_TYPE_MAPPING("hive.server2.table.type.mapping", VarType.STRING, + new StringsValidator("CLASSIC")), + HIVE_SERVER2_SESSION_HOOK("hive.server2.session.hook", true), + HIVE_SERVER2_USE_SSL("hive.server2.use.SSL", VarType.BOOLEAN), + HIVE_SERVER2_SSL_KEYSTORE_PATH("hive.server2.keystore.path", true), + HIVE_SERVER2_SSL_KEYSTORE_PASSWORD("hive.server2.keystore.password", true), + + HIVE_SECURITY_COMMAND_WHITELIST("hive.security.command.whitelist"), + + HIVE_CONF_RESTRICTED_LIST("hive.conf.restricted.list", true), // If this is set all move tasks at the end of a multi-insert query will only begin once all // outputs are ready HIVE_MULTI_INSERT_MOVE_TASKS_SHARE_DEPENDENCIES( - "hive.multi.insert.move.tasks.share.dependencies", false), + "hive.multi.insert.move.tasks.share.dependencies", VarType.BOOLEAN), // If this is set, when writing partitions, the metadata will include the bucketing/sorting // properties with which the data was written if any (this will not overwrite the metadata // inherited from the table if the table is bucketed/sorted) - HIVE_INFER_BUCKET_SORT("hive.exec.infer.bucket.sort", false), + HIVE_INFER_BUCKET_SORT("hive.exec.infer.bucket.sort", VarType.BOOLEAN), // If this is set, when setting the number of reducers for the map reduce task which writes the // final output files, it will choose a number which is a power of two. The number of reducers // may be set to a power of two, only to be followed by a merge task meaning preventing // anything from being inferred. HIVE_INFER_BUCKET_SORT_NUM_BUCKETS_POWER_TWO( - "hive.exec.infer.bucket.sort.num.buckets.power.two", false), + "hive.exec.infer.bucket.sort.num.buckets.power.two", VarType.BOOLEAN), /* The following section contains all configurations used for list bucketing feature.*/ /* This is not for clients. but only for block merge task. */ /* This is used by BlockMergeTask to send out flag to RCFileMergeMapper */ /* about alter table...concatenate and list bucketing case. */ HIVEMERGECURRENTJOBCONCATENATELISTBUCKETING( - "hive.merge.current.job.concatenate.list.bucketing", true), + "hive.merge.current.job.concatenate.list.bucketing", VarType.BOOLEAN), /* This is not for clients. but only for block merge task. */ /* This is used by BlockMergeTask to send out flag to RCFileMergeMapper */ /* about depth of list bucketing. */ HIVEMERGECURRENTJOBCONCATENATELISTBUCKETINGDEPTH( - "hive.merge.current.job.concatenate.list.bucketing.depth", 0), + "hive.merge.current.job.concatenate.list.bucketing.depth", VarType.INT), // Enable list bucketing optimizer. Default value is false so that we disable it by default. - HIVEOPTLISTBUCKETING("hive.optimize.listbucketing", false), + HIVEOPTLISTBUCKETING("hive.optimize.listbucketing", VarType.BOOLEAN), // Allow TCP Keep alive socket option for for HiveServer or a maximum timeout for the socket. - SERVER_READ_SOCKET_TIMEOUT("hive.server.read.socket.timeout", 10), - SERVER_TCP_KEEP_ALIVE("hive.server.tcp.keepalive", true), + SERVER_READ_SOCKET_TIMEOUT("hive.server.read.socket.timeout", VarType.INT), + SERVER_TCP_KEEP_ALIVE("hive.server.tcp.keepalive", VarType.BOOLEAN), // Whether to show the unquoted partition names in query results. - HIVE_DECODE_PARTITION_NAME("hive.decode.partition.name", false), + HIVE_DECODE_PARTITION_NAME("hive.decode.partition.name", VarType.BOOLEAN), //Vectorization enabled - HIVE_VECTORIZATION_ENABLED("hive.vectorized.execution.enabled", false), - HIVE_VECTORIZATION_GROUPBY_CHECKINTERVAL("hive.vectorized.groupby.checkinterval", 100000), - HIVE_VECTORIZATION_GROUPBY_MAXENTRIES("hive.vectorized.groupby.maxentries", 1000000), - HIVE_VECTORIZATION_GROUPBY_FLUSH_PERCENT("hive.vectorized.groupby.flush.percent", (float) 0.1), + HIVE_VECTORIZATION_ENABLED("hive.vectorized.execution.enabled", VarType.BOOLEAN), + HIVE_VECTORIZATION_GROUPBY_CHECKINTERVAL("hive.vectorized.groupby.checkinterval", VarType.INT), + HIVE_VECTORIZATION_GROUPBY_MAXENTRIES("hive.vectorized.groupby.maxentries", VarType.INT), + HIVE_VECTORIZATION_GROUPBY_FLUSH_PERCENT("hive.vectorized.groupby.flush.percent", VarType.FLOAT), - - HIVE_TYPE_CHECK_ON_INSERT("hive.typecheck.on.insert", true), + HIVE_TYPE_CHECK_ON_INSERT("hive.typecheck.on.insert", VarType.BOOLEAN), // none, idonly, traverse, execution - HIVESTAGEIDREARRANGE("hive.stageid.rearrange", "none"), - HIVEEXPLAINDEPENDENCYAPPENDTASKTYPES("hive.explain.dependency.append.tasktype", false), + HIVESTAGEIDREARRANGE("hive.stageid.rearrange"), + HIVEEXPLAINDEPENDENCYAPPENDTASKTYPES("hive.explain.dependency.append.tasktype", VarType.BOOLEAN), - HIVECOUNTERGROUP("hive.counters.group.name", "HIVE") + HIVECOUNTERGROUP("hive.counters.group.name") ; public final String varname; - public final String defaultVal; - public final int defaultIntVal; - public final long defaultLongVal; - public final float defaultFloatVal; - public final Class valClass; - public final boolean defaultBoolVal; - private final VarType type; - private final Validator validator; - ConfVars(String varname, String defaultVal) { - this(varname, defaultVal, null); - } + public String defaultVal; + public int defaultIntVal; + public long defaultLongVal; + public float defaultFloatVal; + public boolean defaultBoolVal; - ConfVars(String varname, String defaultVal, Validator validator) { - this.varname = varname; - this.valClass = String.class; - this.defaultVal = defaultVal; - this.defaultIntVal = -1; - this.defaultLongVal = -1; - this.defaultFloatVal = -1; - this.defaultBoolVal = false; - this.type = VarType.STRING; - this.validator = validator; - } + private String description; - ConfVars(String varname, int defaultVal) { - this(varname, defaultVal, null); + ConfVars(String varname) { + this(varname, VarType.STRING, null); } - ConfVars(String varname, int defaultIntVal, Validator validator) { - this.varname = varname; - this.valClass = Integer.class; - this.defaultVal = Integer.toString(defaultIntVal); - this.defaultIntVal = defaultIntVal; - this.defaultLongVal = -1; - this.defaultFloatVal = -1; - this.defaultBoolVal = false; - this.type = VarType.INT; - this.validator = validator; + ConfVars(String varname, boolean emptyString) { + this(varname, VarType.STRING, null); + defaultVal = emptyString ? "" : null; } - - ConfVars(String varname, long defaultVal) { - this(varname, defaultVal, null); + + ConfVars(String varname, VarType type) { + this(varname, type, null); } - ConfVars(String varname, long defaultLongVal, Validator validator) { + ConfVars(String varname, VarType type, Validator validator) { this.varname = varname; - this.valClass = Long.class; - this.defaultVal = Long.toString(defaultLongVal); - this.defaultIntVal = -1; - this.defaultLongVal = defaultLongVal; - this.defaultFloatVal = -1; - this.defaultBoolVal = false; - this.type = VarType.LONG; + this.type = type; this.validator = validator; } - ConfVars(String varname, float defaultVal) { - this(varname, defaultVal, null); - } - - ConfVars(String varname, float defaultFloatVal, Validator validator) { - this.varname = varname; - this.valClass = Float.class; - this.defaultVal = Float.toString(defaultFloatVal); - this.defaultIntVal = -1; - this.defaultLongVal = -1; - this.defaultFloatVal = defaultFloatVal; - this.defaultBoolVal = false; - this.type = VarType.FLOAT; - this.validator = validator; + private ConfVars setValue(String value) { + switch (type) { + case BOOLEAN: defaultBoolVal = Boolean.valueOf(value); break; + case INT: defaultIntVal = Integer.valueOf(value); break; + case LONG: defaultLongVal = Long.valueOf(value); break; + case FLOAT: defaultFloatVal = Float.valueOf(value); break; + case STRING: defaultVal = value; break; + } + return this; } - ConfVars(String varname, boolean defaultBoolVal) { - this.varname = varname; - this.valClass = Boolean.class; - this.defaultVal = Boolean.toString(defaultBoolVal); - this.defaultIntVal = -1; - this.defaultLongVal = -1; - this.defaultFloatVal = -1; - this.defaultBoolVal = defaultBoolVal; - this.type = VarType.BOOLEAN; - this.validator = null; + private ConfVars setDescription(String description) { + this.description = description; + return this; } public boolean isType(String value) { @@ -967,24 +913,15 @@ public String typeString() { return type.typeString(); } + public String getDescription() { + return description; + } + @Override public String toString() { return varname; } - private static String findHadoopBinary() { - String val = System.getenv("HADOOP_HOME"); - // In Hadoop 1.X and Hadoop 2.X HADOOP_HOME is gone and replaced with HADOOP_PREFIX - if (val == null) { - val = System.getenv("HADOOP_PREFIX"); - } - // and if all else fails we can at least try /usr/bin/hadoop - val = (val == null ? File.separator + "usr" : val) - + File.separator + "bin" + File.separator + "hadoop"; - // Launch hadoop command file on windows. - return val + (Shell.WINDOWS ? ".cmd" : ""); - } - enum VarType { STRING { @Override void checkType(String value) throws Exception { } }, @@ -1038,6 +975,75 @@ private static synchronized InputStream getConfVarInputStream() { return new LoopingByteArrayInputStream(confVarByteArray); } + // copied from hadoop.conf.Configuration + private static void registerConfVars(URL url) { + + HiveConf conf = new HiveConf(); + SystemVariables evaluator = new SystemVariables(); + + InputStream inputStream = null; + try { + inputStream = url.openStream(); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder docBuilder = dbf.newDocumentBuilder(); + Document doc = docBuilder.parse(inputStream); + Element root = doc.getDocumentElement(); + if (!"configuration".equals(root.getTagName())) { + throw new RuntimeException("bad conf file: top-level element not "); + } + NodeList props = root.getChildNodes(); + for (int i = 0; i < props.getLength(); i++) { + Node propNode = props.item(i); + if (!(propNode instanceof Element)) { + continue; + } + Element prop = (Element)propNode; + if (!"property".equals(prop.getTagName())) { + throw new RuntimeException("bad conf file: element not "); + } + NodeList fields = prop.getChildNodes(); + String name = null; + String value = null; + String description = null; + for (int j = 0; j < fields.getLength(); j++) { + Node fieldNode = fields.item(j); + if (!(fieldNode instanceof Element)) { + continue; + } + Element field = (Element)fieldNode; + if ("name".equals(field.getTagName()) && field.hasChildNodes()) { + name = ((Text)field.getFirstChild()).getData().trim(); + } else if ("value".equals(field.getTagName()) && field.hasChildNodes()) { + value = ((Text)field.getFirstChild()).getData(); + } else if ("description".equals(field.getTagName()) && field.hasChildNodes()) { + description = ((Text)field.getFirstChild()).getData(); + } + } + if (name == null) { + throw new RuntimeException("bad conf file: name of property is missing"); + } + ConfVars var = vars.get(name); + if (var == null) { + throw new RuntimeException("bad conf file: invalid name of property " + name); + } + if (value != null && var.type == ConfVars.VarType.STRING) { + value = evaluator.substitute(conf, value, 40); + } + if (value != null) { + var.setValue(value); + } + if (description != null) { + var.setDescription(description); + } + } + } catch (Exception ex) { + ex.printStackTrace(System.err); + throw new RuntimeException(ex); + } finally { + IOUtils.closeStream(inputStream); + } + } + public void verifyAndSet(String name, String value) throws IllegalArgumentException { if (restrictList.contains(name)) { throw new IllegalArgumentException("Cannot modify " + name + " at runtime"); @@ -1046,12 +1052,12 @@ public void verifyAndSet(String name, String value) throws IllegalArgumentExcept } public static int getIntVar(Configuration conf, ConfVars var) { - assert (var.valClass == Integer.class) : var.varname; + assert (var.type == ConfVars.VarType.INT) : var.varname; return conf.getInt(var.varname, var.defaultIntVal); } public static void setIntVar(Configuration conf, ConfVars var, int val) { - assert (var.valClass == Integer.class) : var.varname; + assert (var.type == ConfVars.VarType.INT) : var.varname; conf.setInt(var.varname, val); } @@ -1064,16 +1070,17 @@ public void setIntVar(ConfVars var, int val) { } public static long getLongVar(Configuration conf, ConfVars var) { - assert (var.valClass == Long.class) : var.varname; + assert (var.type == ConfVars.VarType.LONG) : var.varname; return conf.getLong(var.varname, var.defaultLongVal); } public static long getLongVar(Configuration conf, ConfVars var, long defaultVal) { + assert (var.type == ConfVars.VarType.LONG) : var.varname; return conf.getLong(var.varname, defaultVal); } public static void setLongVar(Configuration conf, ConfVars var, long val) { - assert (var.valClass == Long.class) : var.varname; + assert (var.type == ConfVars.VarType.LONG) : var.varname; conf.setLong(var.varname, val); } @@ -1086,16 +1093,17 @@ public void setLongVar(ConfVars var, long val) { } public static float getFloatVar(Configuration conf, ConfVars var) { - assert (var.valClass == Float.class) : var.varname; + assert (var.type == ConfVars.VarType.FLOAT) : var.varname; return conf.getFloat(var.varname, var.defaultFloatVal); } public static float getFloatVar(Configuration conf, ConfVars var, float defaultVal) { + assert (var.type == ConfVars.VarType.FLOAT) : var.varname; return conf.getFloat(var.varname, defaultVal); } public static void setFloatVar(Configuration conf, ConfVars var, float val) { - assert (var.valClass == Float.class) : var.varname; + assert (var.type == ConfVars.VarType.FLOAT) : var.varname; conf.setFloat(var.varname, val); } @@ -1108,16 +1116,17 @@ public void setFloatVar(ConfVars var, float val) { } public static boolean getBoolVar(Configuration conf, ConfVars var) { - assert (var.valClass == Boolean.class) : var.varname; + assert (var.type == ConfVars.VarType.BOOLEAN) : var.varname; return conf.getBoolean(var.varname, var.defaultBoolVal); } public static boolean getBoolVar(Configuration conf, ConfVars var, boolean defaultVal) { + assert (var.type == ConfVars.VarType.BOOLEAN) : var.varname; return conf.getBoolean(var.varname, defaultVal); } public static void setBoolVar(Configuration conf, ConfVars var, boolean val) { - assert (var.valClass == Boolean.class) : var.varname; + assert (var.type == ConfVars.VarType.BOOLEAN) : var.varname; conf.setBoolean(var.varname, val); } @@ -1130,16 +1139,17 @@ public void setBoolVar(ConfVars var, boolean val) { } public static String getVar(Configuration conf, ConfVars var) { - assert (var.valClass == String.class) : var.varname; + assert (var.type == ConfVars.VarType.STRING) : var.varname; return conf.get(var.varname, var.defaultVal); } public static String getVar(Configuration conf, ConfVars var, String defaultVal) { + assert (var.type == ConfVars.VarType.STRING) : var.varname; return conf.get(var.varname, defaultVal); } public static void setVar(Configuration conf, ConfVars var, String val) { - assert (var.valClass == String.class) : var.varname; + assert (var.type == ConfVars.VarType.STRING) : var.varname; conf.set(var.varname, val); } diff --git common/src/java/org/apache/hive/common/util/SystemVariables.java common/src/java/org/apache/hive/common/util/SystemVariables.java new file mode 100644 index 0000000..e3d9b54 --- /dev/null +++ common/src/java/org/apache/hive/common/util/SystemVariables.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.common.util; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; + +public class SystemVariables { + + private static final Log l4j = LogFactory.getLog(SystemVariables.class); + protected static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}"); + + public static final String ENV_PREFIX = "env:"; + public static final String SYSTEM_PREFIX = "system:"; + public static final String HIVECONF_PREFIX = "hiveconf:"; + public static final String HIVEVAR_PREFIX = "hivevar:"; + public static final String SET_COLUMN_NAME = "set"; + + protected String getSubstitute(HiveConf conf, String var) { + String val = null; + try { + if (var.startsWith(SYSTEM_PREFIX)) { + val = System.getProperty(var.substring(SYSTEM_PREFIX.length())); + } + } catch(SecurityException se) { + l4j.warn("Unexpected SecurityException in Configuration", se); + } + if (val == null) { + if (var.startsWith(ENV_PREFIX)) { + val = System.getenv(var.substring(ENV_PREFIX.length())); + } + } + return val; + } + + public String substitute(HiveConf conf, String expr) { + int depth = conf.getIntVar(ConfVars.HIVEVARIABLESUBSTITUTEDEPTH); + return substitute(conf, expr, depth); + } + + public String substitute(HiveConf conf, String expr, int depth) { + Matcher match = varPat.matcher(""); + String eval = expr; + for(int s = 0; s < depth; s++) { + match.reset(eval); + if (!match.find()) { + return eval; + } + String var = match.group(); + var = var.substring(2, var.length()-1); // remove ${ .. } + String val = getSubstitute(conf, var); + + if (val == null) { + l4j.debug("Interpolation result: " + eval); + return eval; // return literal, no substitution found + } + // substitute + eval = eval.substring(0, match.start()) + val + eval.substring(match.end()); + } + throw new IllegalStateException("Variable substitution depth too large: " + + conf.getIntVar(ConfVars.HIVEVARIABLESUBSTITUTEDEPTH) + " " + expr); + } +} diff --git conf/hive-default.xml.template conf/hive-default.xml.template index c61a0bb..6ef007f 100644 --- conf/hive-default.xml.template +++ conf/hive-default.xml.template @@ -79,18 +79,46 @@ + hive.cli.errors.ignore + false + For multiple queries delimited by semicolon in hive client shell, + any single failure stops processing further immediately. If this set true, + hive client ignores failures and proceed to next query. + + + + hive.exec.scratchdir - /tmp/hive-${user.name} + ${system:java.io.tmpdir}/${system:user.name} Scratch space for Hive jobs hive.exec.local.scratchdir - /tmp/${user.name} + ${system:java.io.tmpdir}/${system:user.name} Local scratch space for Hive jobs + hive.scratch.dir.permission + 700 + + + + + hive.exec.submitviachild + false + + + + + hive.mapred.partitioner + org.apache.hadoop.hive.ql.io.DefaultHivePartitioner + + + + + hive.test.mode false whether hive is running in test mode. If yes, it turns on sampling and prefixes the output tablename @@ -122,6 +150,48 @@ + hive.exec.job.debug.timeout + 30000 + + + + + hive.exec.tasklog.debug.timeout + 20000 + + + + + hive.exec.mode.local.auto.inputbytes.max + 134217728 + + + + + hive.exec.mode.local.auto.input.files.max + 4 + + + + + hive.jobname.length + 50 + + + + + hive.jar.path + + + + + + hive.aux.jars.path + + + + + hive.metastore.uris Thrift uri for the remote metastore. Used by metastore client to connect to remote metastore. @@ -368,12 +438,24 @@ + hive.semantic.analyzer.hook + + + + + hive.default.fileformat TextFile Default file format for CREATE TABLE statement. Options are TextFile and SequenceFile. Users can explicitly say CREATE TABLE ... STORED AS <TEXTFILE|SEQUENCEFILE> to override + hive.query.result.fileformat + TextFile + + + + hive.default.rcfile.serde org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe The default SerDe hive will use for the rcfile format @@ -453,6 +535,18 @@ + hive.session.silent + false + whether session is running in silent mode or not + + + + hive.optimize.metadataonly + true + + + + hive.map.aggr.hash.min.reduction 0.5 Hash aggregation will be turned off if the ratio between hash @@ -491,6 +585,12 @@ + hive.ppd.remove.duplicatefilters + true + + + + hive.optimize.groupby true Whether to enable the bucketed group by from bucketed partitions/tables. @@ -541,7 +641,7 @@ - hive.multigroupby.singlemr + hive.multigroupby.singlereducer false Whether to optimize multi group by query to generate single M/R job plan. If the multi group by query has common group by keys, it will be @@ -583,6 +683,12 @@ + hive.outerjoin.supports.filters + true + + + + hive.join.emit.interval 1000 How many rows in the right-most join operand Hive should buffer before emitting the join result. @@ -660,6 +766,18 @@ + hive.exec.script.wrapper + + + + + + hive.exec.script.trust + false + + + + hive.exec.script.maxerrsize 100000 Maximum number of bytes a script is allowed to emit to standard error (per map-reduce task). This prevents runaway scripts from filling logs partitions to capacity @@ -698,6 +816,18 @@ + hive.intermediate.compression.codec + + + + + + hive.intermediate.compression.type + + + + + hive.exec.parallel false Whether to execute jobs in parallel @@ -723,7 +853,7 @@ hive.hwi.war.file - lib/hive-hwi-@VERSION@.war + ${system:HWI_WAR_FILE} This sets the path to the HWI war file, relative to ${HIVE_HOME}. @@ -806,6 +936,37 @@ + hive.merge.rcfile.block.level + true + + + + + hive.merge.input.format.block.level + org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileBlockMergeInputFormat + + + + + hive.merge.current.job.has.dynamic.partitions + false + + + + + hive.merge.current.job.concatenate.list.bucketing + true + + + + + hive.merge.current.job.concatenate.list.bucketing.depth + 0 + + + + + hive.mapjoin.smalltable.filesize 25000000 The threshold for the input file size of the small tables; if the file size is smaller than this threshold, it will try to convert the common join into map join @@ -836,6 +997,24 @@ + hive.hashtable.initialCapacity + 100000 + + + + + hive.hashtable.loadfactor + 0.75 + + + + + hive.debug.localtask + false + + + + hive.auto.convert.join false Whether Hive enable the optimization about converting common join into mapjoin based on the input file size @@ -877,7 +1056,25 @@ Read from a binary stream and treat each hive.binary.record.max.length bytes as a record. The last record before the end of stream can have less than hive.binary.record.max.length bytes - + + + hive.server2.use.SSL + false + + + + + hive.server2.keystore.path + + + + + + hive.server2.keystore.password + + + + hive.server2.transport.mode binary @@ -958,13 +1155,19 @@ hive.querylog.location - /tmp/${user.name} + ${system:java.io.tmpdir}/${system:user.name} Location of Hive run time structured log file + hive.downloaded.resources.dir + ${system:java.io.tmpdir} + + + + hive.querylog.enable.plan.progress true @@ -1001,6 +1204,42 @@ + hive.optimize.sampling.orderby + false + + + + + hive.optimize.sampling.orderby.number + 1000 + + + + + hive.optimize.sampling.orderby.percent + 0.1f + + + + + hive.optimize.listbucketing + false + Enable list bucketing optimizer. Default value is false so that we disable it by default. + + + + hive.optimize.bucketmapjoin + false + optimize bucket map join + + + + hive.optimize.bucketmapjoin.sortedmerge + false + try to use sorted merge bucket map join + + + hive.optimize.bucketingsorting true If hive.enforce.bucketing or hive.enforce.sorting is true, dont create a reducer for enforcing @@ -1299,12 +1538,88 @@ + hive.stats.atomic + false + whether to update metastore stats only if all stats are available + + + + hive.stats.collect.rawdatasize + true + should the raw data size be collected when analyzing tables + + + + hive.stats.max.variable.length + 100 + if length of variable length data type cannot be determined this length will be used. + + + + hive.stats.list.num.entries + 10 + if number of elements in list cannot be determined, this value will be used + + + + hive.stats.map.num.entries + 10 + if number of elements in map cannot be determined, this value will be used + + + + hive.stats.map.parallelism + 1 + to accurately compute statistics for GROUPBY map side parallelism needs to be known + + + + hive.stats.fetch.column.stats + false + statistics annotation fetches column statistics for all required columns and for all required partitions which can be very expensive sometimes + + + + hive.stats.avg.row.size + 10000 + in the absence of table/partition stats, average row size will be used to estimate the number of rows/data size + + + + hive.stats.join.factor + 1.1 + in the absence of column statistics, the estimated number of rows/data size that will emitted from join operator will depend on t factor + + + + hive.stats.deserialization.factor + 1.0 + in the absence of uncompressed/raw data size, total file size will be used for statistics + annotation. But the file may be compressed, encoded and serialized which may be lesser in size + than the actual uncompressed/raw data size. This factor will be multiplied to file size to estimate + the raw data size. + + + + hive.support.concurrency false Whether hive supports concurrency or not. A zookeeper instance must be up and running for the default hive lock manager to support read-write locks. + hive.lock.manager + org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager + + + + + hive.lockmgr.zookeeper.default.partition.name + __HIVE_DEFAULT_ZOOKEEPER_PARTITION__ + + + + hive.lock.numretries 100 The number of times you want to try to get all the locks @@ -1561,6 +1876,12 @@ + hive.optimize.index.autoupdate + false + automatically update stale indexes + + + hive.index.compact.query.max.size 10737418240 The maximum number of bytes that a query using the compact index can read. Negative value is equivalent to infinity. @@ -1748,6 +2069,12 @@ + hive.exec.rcfile.use.sync.cache + true + + + + hive.exec.orc.dictionary.key.size.threshold 0.8 @@ -1757,6 +2084,24 @@ + hive.exec.orc.memory.pool + 0.5 + Maximum fraction of heap that can be used by ORC file writers + + + + hive.exec.orc.write.format + + Define the version of the file to write + + + + hive.exec.orc.default.stripe.size + 268435456 + Define the default ORC stripe size + + + hive.multi.insert.move.tasks.share.dependencies false @@ -1830,6 +2175,16 @@ + hive.hmshandler.force.reload.conf + false + Whether to force reloading of the HMSHandler configuration (including + the connection URL, before the next metastore query that accesses the + datastore. Once reloaded, this value is reset to false. Used for + testing only. + + + + hive.server.read.socket.timeout 10 Timeout for the HiveServer to close the connection if no response from the client in N seconds, defaults to 10 seconds. @@ -1983,7 +2338,7 @@ - >hive.server2.authentication.kerberos.principal + hive.server2.authentication.kerberos.principal Kerberos server principal @@ -1991,7 +2346,7 @@ - >hive.server2.authentication.kerberos.keytab + hive.server2.authentication.kerberos.keytab Kerberos keytab file for server principal @@ -2007,6 +2362,18 @@ + hive.server2.authentication.ldap.Domain + + + + + + hive.server2.session.hook + + + + + hive.server2.authentication.ldap.baseDN @@ -2104,4 +2471,101 @@ + + hive.metastore.fs.handler.class + org.apache.hadoop.hive.metastore.HiveMetaStoreFsImpl + + + + + hive.metastore.expression.proxy + org.apache.hadoop.hive.ql.optimizer.ppr.PartitionExpressionForMetastore + + + + + datanucleus.fixedDatastore + false + + + + + datanucleus.rdbms.useLegacyNativeValueStrategy + true + + + + + hive.metastore.force.reload.conf + false + + + + + hive.metastore.pre.event.listeners + + + + + + hive.metastore.metadb.dir + + + + + + hive.metastore.archive.intermediate.archived + _INTERMEDIATE_ARCHIVED + + + + + hive.metastore.archive.intermediate.original + _INTERMEDIATE_ORIGINAL + + + + + hive.metastore.archive.intermediate.extracted + _INTERMEDIATE_EXTRACTED + + + + + hive.metastore.try.direct.sql.ddl + true + + + + + hive.metastore.try.direct.sql + true + + + + + hive.insert.into.external.tables + + + + + + hive.typecheck.on.insert + true + + + + + hive.stageid.rearrange + none + none, idonly, traverse, execution + + + + hive.explain.dependency.append.tasktype + false + + + + diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 41df473..0fc9875 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -141,6 +141,7 @@ import org.apache.hadoop.hive.ql.plan.RevokeDesc; import org.apache.hadoop.hive.ql.plan.RoleDDLDesc; import org.apache.hadoop.hive.ql.plan.ShowColumnsDesc; +import org.apache.hadoop.hive.ql.plan.ShowConfDesc; import org.apache.hadoop.hive.ql.plan.ShowCreateTableDesc; import org.apache.hadoop.hive.ql.plan.ShowDatabasesDesc; import org.apache.hadoop.hive.ql.plan.ShowFunctionsDesc; @@ -391,6 +392,11 @@ public int execute(DriverContext driverContext) { return showCreateTable(db, showCreateTbl); } + ShowConfDesc showConf = work.getShowConfDesc(); + if (showConf != null) { + return showConf(db, showConf); + } + RoleDDLDesc roleDDLDesc = work.getRoleDDLDesc(); if (roleDDLDesc != null) { return roleDDL(roleDDLDesc); @@ -452,6 +458,29 @@ public int execute(DriverContext driverContext) { return 0; } + private int showConf(Hive db, ShowConfDesc showConf) throws Exception { + ConfVars conf = HiveConf.getConfVars(showConf.getConfName()); + if (conf == null) { + throw new HiveException("invalid configuration name " + showConf.getConfName()); + } + String description = conf.getDescription(); + DataOutputStream output = getOutputStream(showConf.getResFile()); + try { + if (description != null) { + output.write(description.getBytes()); + output.write(terminator); + } + } finally { + output.close(); + } + return 0; + } + + private DataOutputStream getOutputStream(Path outputFile) throws Exception { + FileSystem fs = outputFile.getFileSystem(conf); + return fs.create(outputFile); + } + /** * First, make sure the source table/partition is not * archived/indexes/non-rcfile. If either of these is true, throw an diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 17e6aad..450a741 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -109,6 +109,7 @@ import org.apache.hadoop.hive.ql.plan.RevokeDesc; import org.apache.hadoop.hive.ql.plan.RoleDDLDesc; import org.apache.hadoop.hive.ql.plan.ShowColumnsDesc; +import org.apache.hadoop.hive.ql.plan.ShowConfDesc; import org.apache.hadoop.hive.ql.plan.ShowCreateTableDesc; import org.apache.hadoop.hive.ql.plan.ShowDatabasesDesc; import org.apache.hadoop.hive.ql.plan.ShowFunctionsDesc; @@ -130,9 +131,6 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; @@ -305,6 +303,10 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { ctx.setResFile(new Path(ctx.getLocalTmpFileURI())); analyzeShowDbLocks(ast); break; + case HiveParser.TOK_SHOWCONF: + ctx.setResFile(new Path(ctx.getLocalTmpFileURI())); + analyzeShowConf(ast); + break; case HiveParser.TOK_DESCFUNCTION: ctx.setResFile(new Path(ctx.getLocalTmpFileURI())); analyzeDescFunction(ast); @@ -2318,6 +2320,14 @@ private void analyzeShowDbLocks(ASTNode ast) throws SemanticException { ctx.setNeedLockMgr(true); } + private void analyzeShowConf(ASTNode ast) throws SemanticException { + String confName = stripQuotes(ast.getChild(0).getText()); + ShowConfDesc showConfDesc = new ShowConfDesc(ctx.getResFile(), confName); + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), + showConfDesc), conf)); + setFetchTask(createFetchTask(showConfDesc.getSchema())); + } + /** * Add the task according to the parsed command tree. This is used for the CLI * command "LOCK TABLE ..;". diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g index 366b714..b4590c9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g @@ -263,6 +263,7 @@ KW_USER: 'USER'; KW_ROLE: 'ROLE'; KW_INNER: 'INNER'; KW_EXCHANGE: 'EXCHANGE'; +KW_CONF: 'CONF'; // Operators // NOTE: if you add a new function/operator, add it to sysFuncNames so that describe function _FUNC_ will work. diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index b146df6..5daceea 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -162,6 +162,7 @@ TOK_SHOW_CREATETABLE; TOK_SHOW_TABLESTATUS; TOK_SHOW_TBLPROPERTIES; TOK_SHOWLOCKS; +TOK_SHOWCONF; TOK_LOCKTABLE; TOK_UNLOCKTABLE; TOK_LOCKDB; @@ -1245,6 +1246,8 @@ showStatement | KW_SHOW KW_LOCKS KW_DATABASE (dbName=Identifier) (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWDBLOCKS $dbName $isExtended?) | KW_SHOW (showOptions=KW_FORMATTED)? (KW_INDEX|KW_INDEXES) KW_ON showStmtIdentifier ((KW_FROM|KW_IN) db_name=identifier)? -> ^(TOK_SHOWINDEXES showStmtIdentifier $showOptions? $db_name?) + | KW_SHOW KW_CONF StringLiteral + -> ^(TOK_SHOWCONF StringLiteral) ; lockStatement diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java index 542d59a..a2d1dd6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java @@ -69,6 +69,7 @@ commandType.put(HiveParser.TOK_SHOWPARTITIONS, HiveOperation.SHOWPARTITIONS); commandType.put(HiveParser.TOK_SHOWLOCKS, HiveOperation.SHOWLOCKS); commandType.put(HiveParser.TOK_SHOWDBLOCKS, HiveOperation.SHOWLOCKS); + commandType.put(HiveParser.TOK_SHOWCONF, HiveOperation.SHOWCONF); commandType.put(HiveParser.TOK_CREATEFUNCTION, HiveOperation.CREATEFUNCTION); commandType.put(HiveParser.TOK_DROPFUNCTION, HiveOperation.DROPFUNCTION); commandType.put(HiveParser.TOK_CREATEMACRO, HiveOperation.CREATEMACRO); @@ -189,6 +190,7 @@ public static BaseSemanticAnalyzer get(HiveConf conf, ASTNode tree) case HiveParser.TOK_SHOWINDEXES: case HiveParser.TOK_SHOWLOCKS: case HiveParser.TOK_SHOWDBLOCKS: + case HiveParser.TOK_SHOWCONF: case HiveParser.TOK_CREATEINDEX: case HiveParser.TOK_DROPINDEX: case HiveParser.TOK_ALTERTABLE_CLUSTER_SORT: diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticException.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticException.java index e3dca57..3297de2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticException.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticException.java @@ -40,6 +40,10 @@ public SemanticException(Throwable cause) { super(cause); } + public SemanticException(HiveException cause) { + super(cause); + } + public SemanticException(String message, Throwable cause) { super(message, cause); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java index f292944..69c948c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java @@ -17,79 +17,48 @@ */ package org.apache.hadoop.hive.ql.parse; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.processors.SetProcessor; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hive.common.util.SystemVariables; + +import java.util.Map; -public class VariableSubstitution { +public class VariableSubstitution extends SystemVariables { private static final Log l4j = LogFactory.getLog(VariableSubstitution.class); - protected static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}"); - private String getSubstitute(HiveConf conf, String var) { - String val = null; - try { - if (var.startsWith(SetProcessor.SYSTEM_PREFIX)) { - val = System.getProperty(var.substring(SetProcessor.SYSTEM_PREFIX.length())); - } - } catch(SecurityException se) { - l4j.warn("Unexpected SecurityException in Configuration", se); - } - if (val ==null){ - if (var.startsWith(SetProcessor.ENV_PREFIX)){ - val = System.getenv(var.substring(SetProcessor.ENV_PREFIX.length())); - } - } + @Override + protected String getSubstitute(HiveConf conf, String var) { + String val = super.getSubstitute(conf, var); if (val == null) { - if (var.startsWith(SetProcessor.HIVECONF_PREFIX)){ - val = conf.get(var.substring(SetProcessor.HIVECONF_PREFIX.length())); + if (var.startsWith(HIVECONF_PREFIX)) { + val = conf.get(var.substring(HIVECONF_PREFIX.length())); } } - if (val ==null){ - if(var.startsWith(SetProcessor.HIVEVAR_PREFIX)){ - val = SessionState.get().getHiveVariables().get(var.substring(SetProcessor.HIVEVAR_PREFIX.length())); + if (val == null){ + Map vars = SessionState.get().getHiveVariables(); + if (var.startsWith(HIVEVAR_PREFIX)) { + val = vars.get(var.substring(HIVEVAR_PREFIX.length())); } else { - val = SessionState.get().getHiveVariables().get(var); + val = vars.get(var); } } return val; } + @Override public String substitute (HiveConf conf, String expr) { - - if (conf.getBoolVar(ConfVars.HIVEVARIABLESUBSTITUTE)){ - l4j.debug("Substitution is on: "+expr); + if (conf.getBoolVar(ConfVars.HIVEVARIABLESUBSTITUTE)) { + l4j.debug("Substitution is on: " + expr); } else { return expr; } if (expr == null) { return null; } - Matcher match = varPat.matcher(""); - String eval = expr; - for(int s=0;s inputs, HashSet outputs, this.truncateTblDesc = truncateTblDesc; } + public DDLWork(HashSet inputs, HashSet outputs, + ShowConfDesc showConfDesc) { + this(inputs, outputs); + this.showConfDesc = showConfDesc; + } + public DescDatabaseDesc getDescDatabaseDesc() { return descDbDesc; } @@ -1085,4 +1093,12 @@ public void setAlterTableExchangePartition( AlterTableExchangePartition alterTableExchangePartition) { this.alterTableExchangePartition = alterTableExchangePartition; } + + public ShowConfDesc getShowConfDesc() { + return showConfDesc; + } + + public void setShowConfDesc(ShowConfDesc showConfDesc) { + this.showConfDesc = showConfDesc; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java index bfd6b77..da06746 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java @@ -66,6 +66,7 @@ SHOWINDEXES("SHOWINDEXES", null, null), SHOWPARTITIONS("SHOWPARTITIONS", null, null), SHOWLOCKS("SHOWLOCKS", null, null), + SHOWCONF("SHOWCONF", null, null), CREATEFUNCTION("CREATEFUNCTION", null, null), DROPFUNCTION("DROPFUNCTION", null, null), CREATEMACRO("CREATEMACRO", null, null), diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ShowConfDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/ShowConfDesc.java new file mode 100644 index 0000000..ea4b860 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ShowConfDesc.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.plan; + +import org.apache.hadoop.fs.Path; + +import java.io.Serializable; + +public class ShowConfDesc extends DDLDesc implements Serializable { + private static final long serialVersionUID = 1L; + + private Path resFile; + private String confName; + + /** + * thrift ddl for the result of show columns. + */ + private static final String schema = "Field#string"; + + public String getSchema() { + return schema; + } + + public ShowConfDesc() { + } + + public ShowConfDesc(Path resFile, String confName) { + this.resFile = resFile; + this.confName = confName; + } + + @Explain(displayName = "result file", normalExplain = false) + public Path getResFile() { + return resFile; + } + + public void setResFile(Path resFile) { + this.resFile = resFile; + } + + @Explain(displayName = "conf name", normalExplain = false) + public String getConfName() { + return confName; + } + + public void setConfName(String confName) { + this.confName = confName; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java index c27b689..abc496a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java @@ -22,6 +22,8 @@ import static org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME; import static org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.defaultNullString; +import static org.apache.hive.common.util.SystemVariables.*; + import java.util.Map; import java.util.Properties; import java.util.SortedMap; @@ -39,12 +41,7 @@ */ public class SetProcessor implements CommandProcessor { - private static String prefix = "set: "; - public static final String ENV_PREFIX = "env:"; - public static final String SYSTEM_PREFIX = "system:"; - public static final String HIVECONF_PREFIX = "hiveconf:"; - public static final String HIVEVAR_PREFIX = "hivevar:"; - public static final String SET_COLUMN_NAME = "set"; + private static final String prefix = "set: "; public static boolean getBoolean(String value) { if (value.equals("on") || value.equals("true")) { @@ -69,7 +66,7 @@ private void dumpOptions(Properties p) { // Inserting hive variables for (String s : ss.getHiveVariables().keySet()) { - sortedMap.put(SetProcessor.HIVEVAR_PREFIX + s, ss.getHiveVariables().get(s)); + sortedMap.put(HIVEVAR_PREFIX + s, ss.getHiveVariables().get(s)); } for (Map.Entry entries : sortedMap.entrySet()) { @@ -107,23 +104,23 @@ private CommandProcessorResponse setVariable(String varname, String varvalue){ if (varvalue.contains("\n")){ ss.err.println("Warning: Value had a \\n character in it."); } - if (varname.startsWith(SetProcessor.ENV_PREFIX)){ + if (varname.startsWith(ENV_PREFIX)){ ss.err.println("env:* variables can not be set."); return new CommandProcessorResponse(1); - } else if (varname.startsWith(SetProcessor.SYSTEM_PREFIX)){ - String propName = varname.substring(SetProcessor.SYSTEM_PREFIX.length()); + } else if (varname.startsWith(SYSTEM_PREFIX)){ + String propName = varname.substring(SYSTEM_PREFIX.length()); System.getProperties().setProperty(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue)); return new CommandProcessorResponse(0); - } else if (varname.startsWith(SetProcessor.HIVECONF_PREFIX)){ - String propName = varname.substring(SetProcessor.HIVECONF_PREFIX.length()); + } else if (varname.startsWith(HIVECONF_PREFIX)){ + String propName = varname.substring(HIVECONF_PREFIX.length()); try { setConf(varname, propName, varvalue, false); return new CommandProcessorResponse(0); } catch (IllegalArgumentException e) { return new CommandProcessorResponse(1, e.getMessage(), "42000"); } - } else if (varname.startsWith(SetProcessor.HIVEVAR_PREFIX)) { - String propName = varname.substring(SetProcessor.HIVEVAR_PREFIX.length()); + } else if (varname.startsWith(HIVEVAR_PREFIX)) { + String propName = varname.substring(HIVEVAR_PREFIX.length()); ss.getHiveVariables().put(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue)); return new CommandProcessorResponse(0); } else { @@ -168,7 +165,7 @@ private void setConf(String varname, String key, String varvalue, boolean regist private SortedMap propertiesToSortedMap(Properties p){ SortedMap sortedPropMap = new TreeMap(); - for (Map.Entry entry :System.getProperties().entrySet() ){ + for (Map.Entry entry : p.entrySet() ){ sortedPropMap.put( (String) entry.getKey(), (String) entry.getValue()); } return sortedPropMap; @@ -187,38 +184,38 @@ private CommandProcessorResponse getVariable(String varname){ ss.out.println("silent" + "=" + ss.getIsSilent()); return new CommandProcessorResponse(0); } - if (varname.startsWith(SetProcessor.SYSTEM_PREFIX)){ - String propName = varname.substring(SetProcessor.SYSTEM_PREFIX.length()); + if (varname.startsWith(SYSTEM_PREFIX)){ + String propName = varname.substring(SYSTEM_PREFIX.length()); String result = System.getProperty(propName); if (result != null){ - ss.out.println(SetProcessor.SYSTEM_PREFIX+propName + "=" + result); + ss.out.println(SYSTEM_PREFIX+propName + "=" + result); return new CommandProcessorResponse(0); } else { ss.out.println( propName + " is undefined as a system property"); return new CommandProcessorResponse(1); } - } else if (varname.indexOf(SetProcessor.ENV_PREFIX)==0){ + } else if (varname.indexOf(ENV_PREFIX)==0){ String var = varname.substring(ENV_PREFIX.length()); if (System.getenv(var)!=null){ - ss.out.println(SetProcessor.ENV_PREFIX+var + "=" + System.getenv(var)); + ss.out.println(ENV_PREFIX+var + "=" + System.getenv(var)); return new CommandProcessorResponse(0); } else { ss.out.println(varname + " is undefined as an environmental variable"); return new CommandProcessorResponse(1); } - } else if (varname.indexOf(SetProcessor.HIVECONF_PREFIX)==0) { - String var = varname.substring(SetProcessor.HIVECONF_PREFIX.length()); + } else if (varname.indexOf(HIVECONF_PREFIX)==0) { + String var = varname.substring(HIVECONF_PREFIX.length()); if (ss.getConf().get(var)!=null){ - ss.out.println(SetProcessor.HIVECONF_PREFIX+var + "=" + ss.getConf().get(var)); + ss.out.println(HIVECONF_PREFIX+var + "=" + ss.getConf().get(var)); return new CommandProcessorResponse(0); } else { ss.out.println(varname + " is undefined as a hive configuration variable"); return new CommandProcessorResponse(1); } - } else if (varname.indexOf(SetProcessor.HIVEVAR_PREFIX)==0) { - String var = varname.substring(SetProcessor.HIVEVAR_PREFIX.length()); + } else if (varname.indexOf(HIVEVAR_PREFIX)==0) { + String var = varname.substring(HIVEVAR_PREFIX.length()); if (ss.getHiveVariables().get(var)!=null){ - ss.out.println(SetProcessor.HIVEVAR_PREFIX+var + "=" + ss.getHiveVariables().get(var)); + ss.out.println(HIVEVAR_PREFIX+var + "=" + ss.getHiveVariables().get(var)); return new CommandProcessorResponse(0); } else { ss.out.println(varname + " is undefined as a hive variable"); diff --git ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 0684aac..d1fc7bb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -642,7 +642,7 @@ public static boolean canDownloadResource(String value) { private String downloadResource(String value, boolean convertToUnix) { if (canDownloadResource(value)) { getConsole().printInfo("converting to local " + value); - File resourceDir = new File(getConf().getVar(HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR)); + File resourceDir = new File(getResourceDir()); String destinationName = new Path(value).getName(); File destinationFile = new File(resourceDir, destinationName); if (resourceDir.exists() && ! resourceDir.isDirectory()) { @@ -803,9 +803,8 @@ public void setCurrentDatabase(String currentDatabase) { } public void close() throws IOException { - File resourceDir = - new File(getConf().getVar(HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR)); - LOG.debug("Removing resource dir " + resourceDir); + File resourceDir = new File(getResourceDir()); + LOG.debug("Removing resource dir " + getResourceDir()); try { if (resourceDir.exists()) { FileUtils.deleteDirectory(resourceDir); @@ -815,6 +814,11 @@ public void close() throws IOException { } } + private String getResourceDir() { + String var = getConf().getVar(ConfVars.DOWNLOADED_RESOURCES_DIR); + return var + "/" + getSessionId() + "_resources"; + } + /** * @param resetPerfLogger * @return Tries to return an instance of the class whose name is configured in