Index: src/packages/update-hcatalog-env.sh =================================================================== --- src/packages/update-hcatalog-env.sh (revision 1230784) +++ src/packages/update-hcatalog-env.sh (working copy) @@ -24,14 +24,56 @@ Required parameters: --prefix=PREFIX path to install into - Optional parameters: + Optional installation parameters: --arch=i386 OS Architecture --bin-dir=PREFIX/bin Executable directory --conf-dir=/etc/hcatalog Configuration directory + --lib-dir=/PREFIX/lib Library directory --log-dir=/var/log/hcatalog Log directory - --mysql-dir=/usr/share/java MySQL connector directory --pid-dir=/var/run PID file location --sbin-dir=PREFIX/sbin System executable directory + --java-home=/opt/jre-home JDK or JRE JAVA_HOME + + --proto-env=/path/to/hcat-env.sh.template + path to hcat-env.sh template + --proto-site=/path/to/proto-hive-site.xml + path to hive-site.xml template + --output-env=/path/to/hcat-env.sh + path to file to write out to, in case we + don't want to replace the installed hcat-env.sh + (used only if not specifying --install) + --output-site=/path/to/hive-site.xml + path to file to write out to, in case we + don't want to replace the installed hive-site.xml + (used only if not specifying --install) + + Optional hcatalog parameters: + --sasl-enabled Specify if we're using Secure Hadoop + --metastore-port=9933 Server port the metastore runs on + + Optional hcatalog client parameters: + --metastore-server=localhost + Hostname of server on which the metastore server is running + + Optional hcatalog-server parameters: + --user=hcat User that hcat-server runs as + --warehouse-dir=/user/hcat/warehouse + HDFS path to the hive warehouse dir + --dbhost=DBHOST Hostname of server on which database server is setup + --dbname=hcatmetastoredb Database name + --dbjars=/opt/dbjars/ JDBC connector driver directory + --dbdriver=com.mysql.jdbc.Driver + JDBC connector driver to use + --dbproto=mysql JDBC protocol to talk to driver + --dbuser=DBUSER Database user for hcat-server to use + --dbpasswd=DBPASSWORD Database password for hcat-server to use + --keytab-path=/etc/security/keytab + Location of keytab (used only if sasl-enabled is specified) + --kerberos-principal=PRINCIPAL + Kerberos Principal for metastore server to use + --kerberos-realm=REALM Kerberos Principal autocalculated as hcat/_HOST@REALM + (used only if --kerneros-principal is not specified) + " exit 1 } @@ -52,16 +94,36 @@ OPTS=$(getopt \ -n $0 \ -o '' \ + -l 'prefix:' \ -l 'arch:' \ - -l 'prefix:' \ -l 'bin-dir:' \ -l 'conf-dir:' \ -l 'lib-dir:' \ -l 'log-dir:' \ -l 'pid-dir:' \ - -l 'mysql-dir:' \ -l 'sbin-dir:' \ + -l 'java-home:' \ + -l 'proto-env:' \ + -l 'proto-site:' \ + -l 'output-env:' \ + -l 'output-site:' \ + -l 'sasl-enabled' \ + -l 'metastore-port:' \ + -l 'metastore-server:' \ + -l 'user:' \ + -l 'warehouse-dir:' \ + -l 'dbjars:' \ + -l 'dbproto:' \ + -l 'dbhost:' \ + -l 'dbname:' \ + -l 'dbdriver:' \ + -l 'dbuser:' \ + -l 'dbpasswd:' \ + -l 'keytab-path:' \ + -l 'kerberos-principal:' \ + -l 'kerberos-realm:' \ -l 'uninstall' \ + -l 'install' \ -- "$@") if [ $? != 0 ] ; then @@ -71,36 +133,102 @@ eval set -- "${OPTS}" while true ; do case "$1" in + --prefix) + PREFIX=$2 ; shift 2 + ;; --arch) ARCH=$2 ; shift 2 ;; - --prefix) - PREFIX=$2 ; shift 2 - ;; --bin-dir) BIN_DIR=$2 ; shift 2 ;; - --log-dir) - LOG_DIR=$2 ; shift 2 + --conf-dir) + CONF_DIR=$2 ; shift 2 ;; --lib-dir) LIB_DIR=$2 ; shift 2 ;; - --conf-dir) - CONF_DIR=$2 ; shift 2 + --log-dir) + LOG_DIR=$2 ; shift 2 ;; --pid-dir) PID_DIR=$2 ; shift 2 ;; - --mysql-dir) - MYSQL_DIR=$2 ; shift 2 - ;; --sbin-dir) SBIN_DIR=$2 ; shift 2 ;; + --java-home) + JAVA_HOME=$2 ; shift 2 + ;; + --proto-env) + PROTO_ENV=$2 ; shift 2 + ;; + --proto-site) + PROTO_SITE=$2 ; shift 2 + ;; + --output-env) + OUTPUT_ENV=$2 ; shift 2 + ;; + --output-site) + OUTPUT_SITE=$2 ; shift 2 + ;; + --sasl-enabled) + PARAM_SASL_ENABLED="true"; shift + ;; + --metastore-port) + PARAM_METASTORE_PORT=$2 ; shift 2 + ;; + --metastore-server) + PARAM_METASTORE_SERVER=$2 ; shift 2 + ;; + --user) + SVR_USER=$2 ; shift 2 + ;; + --warehouse-dir) + SVR_WHDIR=$2 ; shift 2 + ;; + --dbhost) + SVR_DBHOST=$2 ; shift 2 + ;; + --dbname) + SVR_DBNAME=$2 ; shift 2 + ;; + --dbdriver) + SVR_DBDRIVER=$2 ; shift 2 + ;; + --dbproto) + SVR_DBPROTO=$2 ; shift 2 + ;; + --dbjars) + SVR_DBJARS=$2 ; shift 2 + ;; + --dbuser) + SVR_DBUSER=$2 ; shift 2 + ;; + --dbpasswd) + SVR_DBPASSWD=$2 ; shift 2 + ;; + --keytab-path) + SVR_KEYTAB_PATH=$2 ; shift 2 + ;; + --kerberos-principal) + SVR_KERBEROS_PRINCIPAL=$2 ; shift 2 + ;; + --kerberos-realm) + SVR_KERBEROS_REALM=$2 ; shift 2 + ;; --uninstall) UNINSTALL=1; shift + # echo [--uninstall] Disabled in script mode + # usage + # exit 1 ;; + --install) + INSTALL=1; shift + # echo [--install] Disabled in script mode + # usage + # exit 1 + ;; --) shift ; break ;; @@ -127,21 +255,66 @@ HCAT_LOG_DIR=${LOG_DIR:-$PREFIX/var/log} HCAT_PID_DIR=${PID_DIR:-$PREFIX/var/run} HCAT_SBIN_DIR=${SBIN_DIR:-$PREFIX/sbin} -DBROOT=${MYSQL_DIR:-/usr/sharejava} + +HCAT_PROTO_ENV=${PROTO_ENV:-$HCAT_PREFIX/share/hcatalog/templates/conf/hcat-env.sh.template} +HCAT_PROTO_SITE=${PROTO_SITE:-$HCAT_PREFIX/share/hcatalog/templates/conf/hive-site.xml.template} + +SASL_ENABLED=${PARAM_SASL_ENABLED:-false} +METASTORE_PORT=${PARAM_METASTORE_PORT:-9933} + +METASTORE_SERVER=${PARAM_METASTORE_SERVER:-`uname -n`} + +USER=${SVR_USR:-hcat} +WHDIR=${SVR_WHDIR:-/WAREHOUSE_DIR} +DBHOST=${SVR_DBHOST:-localhost} +DBNAME=${SVR_DBNAME:-hcatmetastoredb} +DBJARS=${SVR_DBJARS:-/opt/dbjars} +DBPROTO=${SVR_DBPROTO:-mysql} +DBDRIVER=${SVR_DBDRIVER:-com.mysql.jdbc.Driver} +DBUSER=${SVR_DBUSER:-DBUSER} +DBPASSWD=${SVR_DBPASSWD:-DBPASSWD} +KEYTAB_PATH=${SVR_KEYTAB_PATH:-KEYTAB_PATH} + +KERBEROS_REALM=${SVR_KERBEROS_REALM:-REALM} +KERBEROS_PRINCIPAL=${SVR_KERBEROS_PRINCIPAL:-hcat/$METASTORE_SERVER@$KERBEROS_REALM} + + UNINSTALL=${UNINSTALL:-0} +INSTALL=${INSTALL:-0} +if [ $INSTALL == "0" ]; then + HCAT_OUTPUT_ENV=${OUTPUT_ENV:-$HCAT_CONF_DIR/hcat-env.sh} + HCAT_OUTPUT_SITE=${OUTPUT_SITE:-$HCAT_CONF_DIR/hive-site.xml} +else + # If in install mode, ignore --output-env and --output-site. we write to the conf dir + HCAT_OUTPUT_ENV=$HCAT_CONF_DIR/hcat-env.sh + HCAT_OUTPUT_SITE=$HCAT_CONF_DIR/hcat-site.sh +fi + if [ "${ARCH}" != "i386" ]; then HCAT_LIB_DIR=${HCAT_LIB_DIR}64 fi -if [ "${UNINSTALL}" -eq "1" ]; then +if [ -z "${JAVA_HOME}" ]; then + if [ -e /etc/debian_version ]; then + JAVA_HOME=/usr/lib/jvm/java-6-sun/jre + else + JAVA_HOME=/usr/java/default + fi +fi + +if [ $UNINSTALL == "1" ]; then # Remove symlinks if [ "${HCAT_CONF_DIR}" != "${HCAT_PREFIX}/etc/hcatalog" ]; then rm -rf ${HCAT_PREFIX}/etc/hcatalog fi rm -f /etc/default/hcat-env.sh rm -f /etc/profile.d/hcat-env.sh -else + echo Uninstall preparation done. + exit 0 +fi + +if [ $INSTALL == "1" ]; then # Create symlinks if [ "${HCAT_CONF_DIR}" != "${HCAT_PREFIX}/etc/hcatalog" ]; then mkdir -p ${HCAT_PREFIX}/etc @@ -150,12 +323,6 @@ ln -sf ${HCAT_CONF_DIR}/hcat-env.sh /etc/default/hcat-env.sh ln -sf ${HCAT_CONF_DIR}/hcat-env.sh /etc/profile.d/hcat-env.sh - if [ ! -e "${HCAT_CONF_DIR}/proto-hive-site.xml" ]; then - cp ${HCAT_CONF_DIR}/proto-hive-site.xml ${HCAT_CONF_DIR}/hive-site.xml - chown hcat:hadoop ${HCAT_CONF_DIR}/hive-site.xml - chmod 700 ${HCAT_CONF_DIR}/hive-site.xml - fi - mkdir -p ${HCAT_LOG_DIR} chown hcat:hadoop ${HCAT_LOG_DIR} chmod 775 ${HCAT_LOG_DIR} @@ -174,7 +341,13 @@ JAVA_HOME=/usr/java/default fi fi - template_generator ${HCAT_PREFIX}/share/hcatalog/templates/conf/hcat-env.sh.template $TFILE - cp ${TFILE} ${HCAT_CONF_DIR}/hcat-env.sh - rm -f ${TFILE} fi + +TFILE1="/tmp/$(basename $0).$$.1.tmp" +TFILE2="/tmp/$(basename $0).$$.2.tmp" +template_generator $HCAT_PROTO_ENV $TFILE1 +template_generator $HCAT_PROTO_SITE $TFILE2 +cp ${TFILE1} ${HCAT_OUTPUT_ENV} ; chown hcat:hadoop ${HCAT_OUTPUT_ENV} ; chmod 775 ${HCAT_OUTPUT_ENV} +cp ${TFILE2} ${HCAT_OUTPUT_SITE} ; chown hcat:hadoop ${HCAT_OUTPUT_SITE} ; chmod 755 ${HCAT_OUTPUT_SITE} +rm -f ${TFILE1} +rm -f ${TFILE2} Index: src/packages/deb/hcatalog.control/postinst =================================================================== --- src/packages/deb/hcatalog.control/postinst (revision 1230784) +++ src/packages/deb/hcatalog.control/postinst (working copy) @@ -22,4 +22,4 @@ --conf-dir=/etc/hcatalog \ --log-dir=/var/log/hcatalog \ --pid-dir=/var/run/hcatalog \ - --mysql-dir=/usr/share/java + --dbjars=/usr/share/java Index: src/packages/rpm/spec/hcatalog.spec =================================================================== --- src/packages/rpm/spec/hcatalog.spec (revision 1230784) +++ src/packages/rpm/spec/hcatalog.spec (working copy) @@ -133,7 +133,7 @@ --conf-dir=${RPM_INSTALL_PREFIX1} \ --log-dir=${RPM_INSTALL_PREFIX2} \ --pid-dir=${RPM_INSTALL_PREFIX3} \ - --mysql-dir=/usr/share/java + --dbjars=/usr/share/java if [ ! -f ${RPM_INSTALL_PREFIX1}/hive-site.xml ]; then cp ${RPM_INSTALL_PREFIX1}/proto-hive-site.xml ${RPM_INSTALL_PREFIX1}/hive-site.xml Index: src/packages/templates/conf/hive-site.xml.template =================================================================== --- src/packages/templates/conf/hive-site.xml.template (revision 0) +++ src/packages/templates/conf/hive-site.xml.template (revision 0) @@ -0,0 +1,106 @@ + + + + + + + + hive.metastore.local + false + controls whether to connect to remove metastore server or open a new metastore server in Hive Client JVM + + + + javax.jdo.option.ConnectionURL + jdbc:${DBPROTO}://${DBHOST}/${DBNAME}?createDatabaseIfNotExist=true + JDBC connect string for a JDBC metastore + + + + javax.jdo.option.ConnectionDriverName + ${DBDRIVER} + Driver class name for a JDBC metastore + + + + javax.jdo.option.ConnectionUserName + ${DBUSER} + username to use against metastore database + + + + javax.jdo.option.ConnectionPassword + ${DBPASSWD} + password to use against metastore database + + + + hive.metastore.warehouse.dir + ${WHDIR} + location of default database for the warehouse + + + + hive.metastore.sasl.enabled + ${SASL_ENABLED} + If true, the metastore thrift interface will be secured with SASL. Clients must authenticate with Kerberos. + + + + hive.metastore.kerberos.keytab.file + ${KEYTAB_PATH} + The path to the Kerberos Keytab file containing the metastore thrift server's service principal. + + + + hive.metastore.kerberos.principal + ${KERBEROS_PRINCIPAL} + The service principal for the metastore thrift server. The special string _HOST will be replaced automatically with the correct host name. + + + + hive.metastore.cache.pinobjtypes + Table,Database,Type,FieldSchema,Order + List of comma separated metastore object types that should be pinned in the cache + + + + hive.metastore.uris + thrift://${METASTORE_SERVER}:${METASTORE_PORT} + URI for client to contact metastore server + + + + hive.semantic.analyzer.factory.impl + org.apache.hcatalog.cli.HCatSemanticAnalyzerFactory + controls which SemanticAnalyzerFactory implemenation class is used by CLI + + + + hadoop.clientside.fs.operations + true + FS operations are owned by client + + + + hive.metastore.client.socket.timeout + 60 + MetaStore Client socket timeout in seconds + + + Index: src/packages/templates/conf/hcat-env.sh.template =================================================================== --- src/packages/templates/conf/hcat-env.sh.template (revision 1230784) +++ src/packages/templates/conf/hcat-env.sh.template (working copy) @@ -18,6 +18,6 @@ HCAT_PID_DIR=${HCAT_PID_DIR} HCAT_LOG_DIR=${HCAT_LOG_DIR} HCAT_CONF_DIR=${HCAT_CONF_DIR} -DBROOT=${DBROOT} -USER=hcat -METASTORE_PORT=9933 +DBROOT=${DBJARS} +USER=${USER} +METASTORE_PORT=${METASTORE_PORT} Index: bin/hcat-config.sh =================================================================== --- bin/hcat-config.sh (revision 1230784) +++ bin/hcat-config.sh (working copy) @@ -49,11 +49,11 @@ # Allow alternate conf dir location. if [ -e "${HCAT_PREFIX}/conf/hcat-env.sh" ]; then - DEFAULT_CONF_DIR="conf" + DEFAULT_CONF_DIR=${HCAT_PREFIX}/"conf" else - DEFAULT_CONF_DIR="etc/hcatalog" + DEFAULT_CONF_DIR="/etc/hcatalog" fi -HCAT_CONF_DIR="${HCAT_CONF_DIR:-$HCAT_PREFIX/$DEFAULT_CONF_DIR}" +HCAT_CONF_DIR="${HCAT_CONF_DIR:-$DEFAULT_CONF_DIR}" if [ -e $HCAT_PREFIX/bin/hadoop ]; then HADOOP_PREFIX=$HCAT_PREFIX Index: bin/hcat_server.sh =================================================================== --- bin/hcat_server.sh (revision 1230784) +++ bin/hcat_server.sh (working copy) @@ -45,7 +45,7 @@ exit 1 fi - HIVE_SITE_XML=${HCAT_PREFIX}/etc/hcatalog/hive-site.xml + HIVE_SITE_XML=${HCAT_CONF_DIR}/hive-site.xml if [ ! -e $HIVE_SITE_XML ] then echo "Missing hive-site.xml, expected at [$HIVE_SITE_XML]";