Index: conf/proto-hive-site.xml =================================================================== --- conf/proto-hive-site.xml (revision 1171198) +++ conf/proto-hive-site.xml (working copy) @@ -81,7 +81,7 @@ hive.metastore.uris - thrift://SVRHOST:9080 + thrift://SVRHOST:9933 URI for client to contact metastore server Index: src/packages/update-hcatalog-env.sh =================================================================== --- src/packages/update-hcatalog-env.sh (revision 0) +++ src/packages/update-hcatalog-env.sh (revision 0) @@ -0,0 +1,180 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script configures hcat-env.sh and symlinkis directories for +# relocating RPM locations. + +usage() { + echo " +usage: $0 + Required parameters: + --prefix=PREFIX path to install into + + Optional parameters: + --arch=i386 OS Architecture + --bin-dir=PREFIX/bin Executable directory + --conf-dir=/etc/hcatalog Configuration directory + --log-dir=/var/log/hcatalog Log directory + --mysql-dir=/usr/share/java MySQL connector directory + --pid-dir=/var/run PID file location + --sbin-dir=PREFIX/sbin System executable directory + " + exit 1 +} + +template_generator() { + REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})' + cat $1 | + while read line ; do + while [[ "$line" =~ $REGEX ]] ; do + LHS=${BASH_REMATCH[1]} + RHS="$(eval echo "\"$LHS\"")" + line=${line//$LHS/$RHS} + done + echo $line >> $2 + done +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'arch:' \ + -l 'prefix:' \ + -l 'bin-dir:' \ + -l 'conf-dir:' \ + -l 'lib-dir:' \ + -l 'log-dir:' \ + -l 'pid-dir:' \ + -l 'mysql-dir:' \ + -l 'sbin-dir:' \ + -l 'uninstall' \ + -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "${OPTS}" +while true ; do + case "$1" in + --arch) + ARCH=$2 ; shift 2 + ;; + --prefix) + PREFIX=$2 ; shift 2 + ;; + --bin-dir) + BIN_DIR=$2 ; shift 2 + ;; + --log-dir) + LOG_DIR=$2 ; shift 2 + ;; + --lib-dir) + LIB_DIR=$2 ; shift 2 + ;; + --conf-dir) + CONF_DIR=$2 ; shift 2 + ;; + --pid-dir) + PID_DIR=$2 ; shift 2 + ;; + --mysql-dir) + MYSQL_DIR=$2 ; shift 2 + ;; + --sbin-dir) + SBIN_DIR=$2 ; shift 2 + ;; + --uninstall) + UNINSTALL=1; shift + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in PREFIX; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing param: $var + usage + fi +done + +ARCH=${ARCH:-i386} +HCAT_PREFIX=$PREFIX +HCAT_BIN_DIR=${BIN_DIR:-$PREFIX/bin} +HCAT_CONF_DIR=${CONF_DIR:-$PREFIX/etc/hcatalog} +HCAT_LIB_DIR=${LIB_DIR:-$PREFIX/lib} +HCAT_LOG_DIR=${LOG_DIR:-$PREFIX/var/log} +HCAT_PID_DIR=${PID_DIR:-$PREFIX/var/run} +HCAT_SBIN_DIR=${SBIN_DIR:-$PREFIX/sbin} +DBROOT=${MYSQL_DIR:-/usr/sharejava} +UNINSTALL=${UNINSTALL:-0} + +if [ "${ARCH}" != "i386" ]; then + HCAT_LIB_DIR=${HCAT_LIB_DIR}64 +fi + +if [ "${UNINSTALL}" -eq "1" ]; then + # Remove symlinks + if [ "${HCAT_CONF_DIR}" != "${HCAT_PREFIX}/etc/hcatalog" ]; then + rm -rf ${HCAT_PREFIX}/etc/hcatalog + fi + rm -f /etc/default/hcat-env.sh + rm -f /etc/profile.d/hcat-env.sh +else + # Create symlinks + if [ "${HCAT_CONF_DIR}" != "${HCAT_PREFIX}/etc/hcatalog" ]; then + mkdir -p ${HCAT_PREFIX}/etc + ln -sf ${HCAT_CONF_DIR} ${HCAT_PREFIX}/etc/hcatalog + fi + ln -sf ${HCAT_CONF_DIR}/hcat-env.sh /etc/default/hcat-env.sh + ln -sf ${HCAT_CONF_DIR}/hcat-env.sh /etc/profile.d/hcat-env.sh + + if [ ! -e "${HCAT_CONF_DIR}/proto-hive-site.xml" ]; then + cp ${HCAT_CONF_DIR}/proto-hive-site.xml ${HCAT_CONF_DIR}/hive-site.xml + chown hcat:hadoop ${HCAT_CONF_DIR}/hive-site.xml + chmod 700 ${HCAT_CONF_DIR}/hive-site.xml + fi + + mkdir -p ${HCAT_LOG_DIR} + chown hcat:hadoop ${HCAT_LOG_DIR} + chmod 775 ${HCAT_LOG_DIR} + + if [ ! -d ${HCAT_PID_DIR} ]; then + mkdir -p ${HCAT_PID_DIR} + chown hcat:hadoop ${HCAT_PID_DIR} + chmod 775 ${HCAT_PID_DIR} + fi + + TFILE="/tmp/$(basename $0).$$.tmp" + if [ -z "${JAVA_HOME}" ]; then + if [ -e /etc/debian_version ]; then + JAVA_HOME=/usr/lib/jvm/java-6-sun/jre + else + JAVA_HOME=/usr/java/default + fi + fi + template_generator ${HCAT_PREFIX}/share/hcatalog/templates/conf/hcat-env.sh.template $TFILE + cp ${TFILE} ${HCAT_CONF_DIR}/hcat-env.sh + rm -f ${TFILE} +fi Index: src/packages/deb/hcatalog.control/control =================================================================== --- src/packages/deb/hcatalog.control/control (revision 0) +++ src/packages/deb/hcatalog.control/control (revision 0) @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +Package: hcatalog +Version: @version@ +Section: misc +Priority: optional +Provides: hcatalog +Architecture: all +Depends: sun-java6-jre, hadoop, mysql-client +Maintainer: Apache Software Foundation +Description: The Apache HCatalog is a table and storage management service for data created using Apache Hadoop. +Distribution: development Index: src/packages/deb/hcatalog.control/postinst =================================================================== --- src/packages/deb/hcatalog.control/postinst (revision 0) +++ src/packages/deb/hcatalog.control/postinst (revision 0) @@ -0,0 +1,25 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bash /usr/sbin/update-hcatalog-env.sh \ + --prefix=/usr \ + --bin-dir=/usr/bin \ + --sbin-dir=/usr/sbin \ + --conf-dir=/etc/hcatalog \ + --log-dir=/var/log/hcatalog \ + --pid-dir=/var/run/hcatalog \ + --mysql-dir=/usr/share/java Index: src/packages/deb/hcatalog.control/preinst =================================================================== --- src/packages/deb/hcatalog.control/preinst (revision 0) +++ src/packages/deb/hcatalog.control/preinst (revision 0) @@ -0,0 +1,20 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -r hadoop + +/usr/sbin/useradd --comment "HCatalog" --shell /bin/bash -M -r -g hadoop --home /usr/share/hcatalog hcat 2> /dev/null || : Index: src/packages/deb/hcatalog.control/postrm =================================================================== --- src/packages/deb/hcatalog.control/postrm (revision 0) +++ src/packages/deb/hcatalog.control/postrm (revision 0) @@ -0,0 +1,18 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exit 0 Index: src/packages/deb/hcatalog.control/prerm =================================================================== --- src/packages/deb/hcatalog.control/prerm (revision 0) +++ src/packages/deb/hcatalog.control/prerm (revision 0) @@ -0,0 +1,25 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bash /usr/sbin/update-hcatalog-env.sh \ + --prefix=/usr \ + --bin-dir=/usr/bin \ + --sbin-dir=/usr/sbin \ + --conf-dir=/etc/hcatalog \ + --log-dir=/var/log/hcatalog \ + --pid-dir=/var/run/hcatalog \ + --uninstal Index: src/packages/deb/hcatalog.control/server.control =================================================================== --- src/packages/deb/hcatalog.control/server.control (revision 0) +++ src/packages/deb/hcatalog.control/server.control (revision 0) @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +Package: hcatalog-server +Version: @version@ +Section: misc +Priority: optional +Provides: hcatalog-server +Architecture: all +Depends: hadoop, mysql-server, libmysql-java, hcatalog +Maintainer: Apache Software Foundation +Description: The Apache HCatalog Metadata Store for data created using Apache Hadoop. +Distribution: development Index: src/packages/deb/hcatalog.control/conffile =================================================================== --- src/packages/deb/hcatalog.control/conffile (revision 0) +++ src/packages/deb/hcatalog.control/conffile (revision 0) @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +/etc/hcatalog Index: src/packages/deb/init.d/hcatalog-server =================================================================== --- src/packages/deb/init.d/hcatalog-server (revision 0) +++ src/packages/deb/init.d/hcatalog-server (revision 0) @@ -0,0 +1,139 @@ +#! /bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### BEGIN INIT INFO +# Provides: hcatalog-server +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: +# Short-Description: Apache HCatalog Server +### END INIT INFO + +set -e + +# /etc/init.d/hcatalog-server: start and stop the Apache HCatalog Server daemon + +test -x /usr/sbin/hcat_server.sh || exit 0 +( /usr/bin/hcat.sh 2>&1 | grep -q hcat ) 2>/dev/null || exit 0 + +umask 022 + +. /lib/lsb/init-functions +. /etc/hcatalog/hcat-env.sh + +# Are we running from init? +run_by_init() { + ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ] +} + +check_for_no_start() { + # forget it if we're trying to start, and /etc/hcatalog/hcatalog-server_not_to_be_run exists + if [ -e /etc/hcatalog/hcatalog-server_not_to_be_run ]; then + if [ "$1" = log_end_msg ]; then + log_end_msg 0 + fi + if ! run_by_init; then + log_action_msg "Apache HCatalog server not in use (/etc/hcatalog/hcatalog-server_not_to_be_run)" + fi + exit 0 + fi +} + +check_privsep_dir() { + # Create the PrivSep empty dir if necessary + if [ ! -d ${HCAT_PID_DIR} ]; then + mkdir -p ${HCAT_PID_DIR} + chown root:hadoop ${HCAT_PID_DIR} + chmod 0775 ${HCAT_PID_DIR} + fi +} + +export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin" + +case "$1" in + start) + check_privsep_dir + check_for_no_start + log_daemon_msg "Starting Apache HCatalog Server" "hcatalog-server" + if start-stop-daemon --start --quiet --oknodo --pidfile ${HCAT_PID_DIR}/hcat.pid -c hcat -x /usr/sbin/hcat_server.sh -- --config ${HCAT_CONF_DIR} start; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + stop) + log_daemon_msg "Stopping Apache HCatalog Server" "hcatalog-server" + if start-stop-daemon --stop --quiet --oknodo --pidfile ${HCAT_PID_DIR}/hcat.pid; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + + restart) + check_privsep_dir + log_daemon_msg "Restarting Apache HCatalog Server" "hcatalog-server" + start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HCAT_PID_DIR}/hcat.pid + check_for_no_start log_end_msg + if start-stop-daemon --start --quiet --oknodo --pidfile ${HCAT_PID_DIR}/hcat.pid -c hcat -x /usr/sbin/hcat_server.sh -- --config ${HCAT_CONF_DIR} start; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + + try-restart) + check_privsep_dir + log_daemon_msg "Restarting Apache HCatalog Server" "hcatalog-server" + set +e + start-stop-daemon --stop --quiet --retry 30 --pidfile ${HCAT_PID_DIR}/hcat.pid + RET="$?" + set -e + case $RET in + 0) + # old daemon stopped + check_for_no_start log_end_msg + if start-stop-daemon --start --quiet --oknodo --pidfile ${HCAT_PID_DIR}/hcat.pid -c hcat -x /usr/sbin/hcat_server.sh -- --config ${HADOOP_CONF_DIR} start; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + 1) + # daemon not running + log_progress_msg "(not running)" + log_end_msg 0 + ;; + *) + # failed to stop + log_progress_msg "(failed to stop)" + log_end_msg 1 + ;; + esac + ;; + + status) + status_of_proc -p ${HCAT_PID_DIR}/hcat.pid ${JAVA_HOME}/bin/java hcat-server && exit 0 || exit $? + ;; + + *) + log_action_msg "Usage: /etc/init.d/hcatalog-server {start|stop|restart|try-restart|status}" + exit 1 +esac + +exit 0 Index: src/packages/rpm/init.d/hcatalog-server =================================================================== --- src/packages/rpm/init.d/hcatalog-server (revision 0) +++ src/packages/rpm/init.d/hcatalog-server (revision 0) @@ -0,0 +1,86 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Starts a HCatalog Server +# +# chkconfig: 2345 95 10 +# description: HCatalog Server + +source /etc/rc.d/init.d/functions +source /etc/hcatalog/hcat-env.sh + +HCAT_PREFIX=/usr + +RETVAL=0 +PIDFILE="${HCAT_PID_DIR}/hcat.pid" +desc="HCatalog Server daemon" + +start() { + echo -n $"Starting $desc (hcatalog-server): " + daemon --user hcat ${HCAT_PREFIX}/sbin/hcat_server.sh --config "${HCAT_CONF_DIR}" start + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hcatalog-server + return $RETVAL +} + +stop() { + echo -n $"Stopping $desc (hcatalog-server): " + daemon --user hcat ${HCAT_PREFIX}/sbin/hcat_server.sh --config "${HCAT_CONF_DIR}" stop + RETVAL=$? + sleep 5 + echo + [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/hcatalog-server $PIDFILE +} + +restart() { + stop + start +} + +checkstatus(){ + status -p $PIDFILE ${JAVA_HOME}/bin/java + RETVAL=$? +} + +condrestart(){ + [ -e /var/lock/subsys/hcatalog-server ] && restart || : +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + checkstatus + ;; + restart) + restart + ;; + condrestart) + condrestart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart}" + exit 1 +esac + +exit $RETVAL Index: src/packages/rpm/spec/hcatalog.spec =================================================================== --- src/packages/rpm/spec/hcatalog.spec (revision 0) +++ src/packages/rpm/spec/hcatalog.spec (revision 0) @@ -0,0 +1,173 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# RPM Spec file for Hcatalog version @version@ +# + +%define name hcatalog +%define version @version@ +%define release @package.release@ + +# Installation Locations +%define _prefix @package.prefix@ +%define _bin_dir %{_prefix}/bin +%define _conf_dir @package.conf.dir@ +%define _lib_dir %{_prefix}/lib +%define _lib64_dir %{_prefix}/lib64 +%define _libexec_dir %{_prefix}/libexec +%define _log_dir @package.log.dir@ +%define _pid_dir @package.pid.dir@ +%define _sbin_dir %{_prefix}/sbin +%define _share_dir %{_prefix}/share +%define _var_dir @package.var.dir@ + +# Build time settings +%define _build_dir @package.build.dir@ +%define _final_name @final.name@ +%define debug_package %{nil} + +# Disable brp-java-repack-jars +%define __os_install_post \ + /usr/lib/rpm/redhat/brp-compress \ + %{!?__debug_package:/usr/lib/rpm/redhat/brp-strip %{__strip}} \ + /usr/lib/rpm/redhat/brp-strip-static-archive %{__strip} \ + /usr/lib/rpm/redhat/brp-strip-comment-note %{__strip} %{__objdump} \ + /usr/lib/rpm/brp-python-bytecompile %{nil} + +# RPM searches perl files for dependancies and this breaks for non packaged perl lib +# like thrift so disable this +%define _use_internal_dependency_generator 0 + +Summary: Apache HCatalog is a table and storage management service for data created using Apache Hadoop. +License: Apache License, Version 2.0 +URL: http://incubator.apache.org/hcatalog +Vendor: Apache Software Foundation +Group: Development/Libraries +Name: %{name} +Version: %{version} +Release: %{release} +Source0: %{_final_name}.tar.gz +Prefix: %{_prefix} +Prefix: %{_conf_dir} +Prefix: %{_log_dir} +Prefix: %{_pid_dir} +Buildroot: %{_build_dir} +Requires: sh-utils, textutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service +AutoReqProv: no +Provides: hcatalog + +%description +The Apache HCatalog is a table and storage management service for data created using Apache Hadoop. + +%prep +%setup -n %{_final_name} + +%build +if [ -d ${RPM_BUILD_DIR}%{_prefix} ]; then + rm -rf ${RPM_BUILD_DIR}%{_prefix} +fi + +if [ -d ${RPM_BUILD_DIR}%{_log_dir} ]; then + rm -rf ${RPM_BUILD_DIR}%{_log_dir} +fi + +if [ -d ${RPM_BUILD_DIR}%{_conf_dir} ]; then + rm -rf ${RPM_BUILD_DIR}%{_conf_dir} +fi + +if [ -d ${RPM_BUILD_DIR}%{_pid_dir} ]; then + rm -rf ${RPM_BUILD_DIR}%{_pid_dir} +fi + +mkdir -p ${RPM_BUILD_DIR}%{_prefix} +mkdir -p ${RPM_BUILD_DIR}%{_bin_dir} +mkdir -p ${RPM_BUILD_DIR}%{_lib_dir} +%ifarch amd64 x86_64 +mkdir -p ${RPM_BUILD_DIR}%{_lib64_dir} +%endif +mkdir -p ${RPM_BUILD_DIR}%{_libexec_dir} +mkdir -p ${RPM_BUILD_DIR}%{_log_dir} +mkdir -p ${RPM_BUILD_DIR}%{_conf_dir} +mkdir -p ${RPM_BUILD_DIR}%{_pid_dir} +mkdir -p ${RPM_BUILD_DIR}%{_sbin_dir} +mkdir -p ${RPM_BUILD_DIR}%{_share_dir} +mkdir -p ${RPM_BUILD_DIR}%{_var_dir} +mkdir -p ${RPM_BUILD_DIR}/etc/init.d + +cp ${RPM_BUILD_DIR}/%{_final_name}/src/packages/rpm/init.d/hcatalog-server ${RPM_BUILD_DIR}/etc/init.d/hcatalog-server +chmod 0755 ${RPM_BUILD_DIR}/etc/init.d/hcatalog-server + +######################### +#### INSTALL SECTION #### +######################### +%install +mv ${RPM_BUILD_DIR}/%{_final_name}/bin/* ${RPM_BUILD_DIR}%{_bin_dir} +mv ${RPM_BUILD_DIR}/%{_final_name}/libexec/* ${RPM_BUILD_DIR}%{_libexec_dir} +mv ${RPM_BUILD_DIR}/%{_final_name}/sbin/* ${RPM_BUILD_DIR}%{_sbin_dir} +mv ${RPM_BUILD_DIR}/%{_final_name}/etc/hcatalog/* ${RPM_BUILD_DIR}%{_conf_dir} +mv ${RPM_BUILD_DIR}/%{_final_name}/share/* ${RPM_BUILD_DIR}%{_share_dir} +rm -rf ${RPM_BUILD_DIR}/%{_final_name} + +%pre +getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -r hadoop +/usr/sbin/useradd --comment "HCatalog" --shell /bin/bash -M -r -g hadoop --home /usr/share/hcatalog hcat 2> /dev/null || : + +%post +bash ${RPM_INSTALL_PREFIX0}/sbin/update-hcatalog-env.sh \ + --prefix=${RPM_INSTALL_PREFIX0} \ + --bin-dir=${RPM_INSTALL_PREFIX0}/bin \ + --sbin-dir=${RPM_INSTALL_PREFIX0}/sbin \ + --conf-dir=${RPM_INSTALL_PREFIX1} \ + --log-dir=${RPM_INSTALL_PREFIX2} \ + --pid-dir=${RPM_INSTALL_PREFIX3} \ + --mysql-dir=/usr/share/java + +if [ ! -f ${RPM_INSTALL_PREFIX1}/hive-site.xml ]; then + cp ${RPM_INSTALL_PREFIX1}/proto-hive-site.xml ${RPM_INSTALL_PREFIX1}/hive-site.xml +fi + +echo HCatalog installed, please take a moment to verify config in ${RPM_INSTALL_PREFIX1}/hcat-env.sh and ${RPM_INSTALL_PREFIX1}/hive-site.xml +%preun +bash ${RPM_INSTALL_PREFIX0}/sbin/update-hcatalog-env.sh \ + --prefix=${RPM_INSTALL_PREFIX0} \ + --bin-dir=${RPM_INSTALL_PREFIX0}/bin \ + --sbin-dir=${RPM_INSTALL_PREFIX0}/sbin \ + --conf-dir=${RPM_INSTALL_PREFIX1} \ + --log-dir=${RPM_INSTALL_PREFIX2} \ + --pid-dir=${RPM_INSTALL_PREFIX3} \ + --uninstall + +%files +%defattr(-,hcat,hadoop) +%attr(0755,hcat,hadoop) %{_log_dir} +%attr(0775,hcat,hadoop) %{_pid_dir} +%config(noreplace) %{_conf_dir} +%{_prefix} +%exclude %{_prefix}/sbin/hcat_server.sh +%exclude %{_prefix}/share/hcatalog/hcatalog-server-extensions* + +%package server +Summary: HCatalog Server +Group: System/Daemons +Requires: hcatalog + +%description server +HCatalog Metadata Store Server + +%files server +%attr(0775,root,hadoop) /etc/init.d/hcatalog-server +%{_prefix}/sbin/hcat_server.sh +%{_prefix}/share/hcatalog/hcatalog-server-extensions* Index: src/packages/templates/conf/hcat-env.sh.template =================================================================== --- src/packages/templates/conf/hcat-env.sh.template (revision 0) +++ src/packages/templates/conf/hcat-env.sh.template (revision 0) @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +JAVA_HOME=${JAVA_HOME} +HCAT_PID_DIR=${HCAT_PID_DIR} +HCAT_LOG_DIR=${HCAT_LOG_DIR} +HCAT_CONF_DIR=${HCAT_CONF_DIR} +DBROOT=${DBROOT} +USER=hcat +METASTORE_PORT=9933 Index: build.xml =================================================================== --- build.xml (revision 1171198) +++ build.xml (working copy) @@ -26,9 +26,12 @@ ================================================================================ --> - + + + - + + @@ -36,6 +39,7 @@ + @@ -84,6 +88,17 @@ + + + + + + + + + + + @@ -198,6 +213,14 @@ + + + + + + - + @@ -247,7 +270,7 @@ --> - @@ -372,26 +395,30 @@ --> - - + + - - - - - - - + + + + + + + + + - - - + + + + + @@ -453,43 +480,65 @@ - - - - - - + + - + + + + + + + + + + + + + + + + + + + + + - + - + - + + + + + + + - + @@ -502,28 +551,30 @@ - - + + + - + - + - + + @@ -538,6 +589,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + @@ -546,11 +622,134 @@ + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: ivy.xml =================================================================== --- ivy.xml (revision 1171198) +++ ivy.xml (working copy) @@ -33,6 +33,7 @@ extends="runtime" description="artifacts needed to compile/test the application"/> + @@ -48,5 +49,6 @@ - + + Index: ivy/libraries.properties =================================================================== --- ivy/libraries.properties (revision 1171198) +++ ivy/libraries.properties (working copy) @@ -22,3 +22,4 @@ activemq.version=5.5.0 javax-mgmt.version=1.1-rev-1 rats-lib.version=0.5.1 +jdeb.version=0.8 Index: bin/hcat =================================================================== --- bin/hcat (revision 1171198) +++ bin/hcat (working copy) @@ -34,10 +34,14 @@ script=`basename "$this"` bin=`unset CDPATH; cd "$bin"; pwd` this="$bin/$script" - -# the root of the HCatalog installation -export HCAT_HOME=`dirname "$this"`/.. + +if [ -e "$bin/../libexec/hcat-config.sh" ]; then + . "$bin"/../libexec/hcat-config.sh +else + . "$bin"/hcat-config.sh +fi + # filter debug command line parameter debug=false @@ -51,25 +55,25 @@ # Find our hcatalog jar shopt -s extglob -HCAT_JAR=$HCAT_HOME/lib/hcatalog-!(*server-extensions*).jar +HCAT_JAR=$HCAT_PREFIX/share/hcatalog/hcatalog-!(*server-extensions*).jar # Add all of the other jars to our classpath -for jar in $HCAT_HOME/lib/*.jar ; do +for jar in $HCAT_PREFIX/share/hcatalog/lib/*.jar ; do HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$jar done # Put our config file in the classpath -HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:${HCAT_HOME}/conf +HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:${HCAT_PREFIX}/etc/hcatalog export HADOOP_CLASSPATH=$HADOOP_CLASSPATH - +export HADOOP_OPTS=$HADOOP_OPTS # run it if [ "$debug" == "true" ]; then echo "Would run:" - echo "exec $HADOOP_HOME/bin/hadoop jar $HCAT_JAR org.apache.hcatalog.cli.HCatCli $remaining" + echo "exec $HADOOP_PREFIX/bin/hadoop jar $HCAT_JAR org.apache.hcatalog.cli.HCatCli $remaining" echo "with HADOOP_CLASSPATH set to ($HADOOP_CLASSPATH)" echo "and HADOOP_OPTS set to ($HADOOP_OPTS)" else - exec $HADOOP_HOME/bin/hadoop jar $HCAT_JAR org.apache.hcatalog.cli.HCatCli "$@" + exec $HADOOP_PREFIX/bin/hadoop jar $HCAT_JAR org.apache.hcatalog.cli.HCatCli "$@" fi Index: bin/hive-config.sh =================================================================== --- bin/hive-config.sh (revision 0) +++ bin/hive-config.sh (revision 0) @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# processes --config option from command line +# + +this="$0" +while [ -h "$this" ]; do + ls=`ls -ld "$this"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '.*/.*' > /dev/null; then + this="$link" + else + this=`dirname "$this"`/"$link" + fi +done + +# convert relative path to absolute path +bin=`dirname "$this"` +script=`basename "$this"` +bin=`cd "$bin"; pwd` +this="$bin/$script" + +# the root of the hcatalog-hive installation +export HIVE_HOME=`dirname "$bin"`/share/hcatalog + +#check to see if the conf dir is given as an optional argument +while [ $# -gt 0 ]; do # Until you run out of parameters . . . + case "$1" in + --config) + shift + confdir=$1 + shift + HIVE_CONF_DIR=$confdir + ;; + --auxpath) + shift + HIVE_AUX_JARS_PATH=$1 + shift + ;; + *) + break; + ;; + esac +done + + +# Allow alternate conf dir location. +HIVE_CONF_DIR="${HIVE_CONF_DIR:-$HIVE_HOME/conf}" + +export HIVE_CONF_DIR=$HIVE_CONF_DIR +export HIVE_AUX_JARS_PATH=$HIVE_AUX_JARS_PATH + +# Default to use 256MB +export HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-256} + + Index: bin/hcat-config.sh =================================================================== --- bin/hcat-config.sh (revision 0) +++ bin/hcat-config.sh (revision 0) @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# included in all the hadoop scripts with source command +# should not be executable directly +# also should not be passed any arguments, since we need original $* + +# resolve links - $0 may be a softlink + +this="${BASH_SOURCE-$0}" +common_bin=$(cd -P -- "$(dirname -- "$this")" && pwd -P) +script="$(basename -- "$this")" +this="$common_bin/$script" + +# convert relative path to absolute path +config_bin=`dirname "$this"` +script=`basename "$this"` +config_bin=`cd "$config_bin"; pwd` +this="$config_bin/$script" + +# the root of the HCATALOG installation +export HCAT_PREFIX=`dirname "$this"`/.. + +#check to see if the conf dir is given as an optional argument +if [ $# -gt 1 ] +then + if [ "--config" = "$1" ] + then + shift + confdir=$1 + shift + HCAT_CONF_DIR=$confdir + fi +fi + +# Allow alternate conf dir location. +if [ -e "${HCAT_PREFIX}/conf/hcat-env.sh" ]; then + DEFAULT_CONF_DIR="conf" +else + DEFAULT_CONF_DIR="etc/hcatalog" +fi +HCAT_CONF_DIR="${HCAT_CONF_DIR:-$HCAT_PREFIX/$DEFAULT_CONF_DIR}" + +if [ -e $HCAT_PREFIX/bin/hadoop ]; then + HADOOP_PREFIX=$HCAT_PREFIX +elif [ -e $HADOOP_HOME/bin/hadoop ]; then + HADOOP_PREFIX=$HADOOP_HOME +else + echo "Hadoop not found." + exit 1 +fi + +if [ -f "${HCAT_CONF_DIR}/hcat-env.sh" ]; then + . "${HCAT_CONF_DIR}/hcat-env.sh" +fi Index: bin/hcat_server.sh =================================================================== --- bin/hcat_server.sh (revision 0) +++ bin/hcat_server.sh (revision 0) @@ -0,0 +1,147 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin=`which $0` +bin=`dirname ${bin}` +bin=`cd "$bin"; pwd` + +if [ -e "$bin/../libexec/hcat-config.sh" ]; then + . "$bin"/../libexec/hcat-config.sh +else + . "$bin"/hcat-config.sh +fi + +function print_usage() { + echo "Usage: $0 [--config confdir] COMMAND" + echo " start Start HCatalog Server" + echo " stop Stop HCatalog Server" +} + +function start_hcat() { + # back ground the metastore service and record the pid + PID_FILE=${HCAT_PID_DIR}/hcat.pid + SLEEP_TIME_AFTER_START=15 + + # check if service is already running, if so exit + if [ -s "$PID_FILE" ] + then + echo "HCatalog server appears to be running. If you are SURE it is not" \ + " remove $PID_FILE and re-run this script." + exit 1 + fi + + HIVE_SITE_XML=${HCAT_PREFIX}/etc/hcatalog/hive-site.xml + if [ ! -e $HIVE_SITE_XML ] + then + echo "Missing hive-site.xml, expected at [$HIVE_SITE_XML]"; + exit 1 + fi + + # Find our Warehouse dir from the config file +# WAREHOUSE_DIR=`sed -n '/hive.metastore.warehouse.dir<\/name>/ { +# n +# s/.*\(.*\)<\/value>.*/\1/p +# }' $HIVE_SITE_XML` +# HADOOP_OPTS="$HADOOP_OPTS -Dhive.metastore.warehouse.dir=$WAREHOUSE_DIR " + + # add in hive-site.xml to classpath + AUX_CLASSPATH=${AUX_CLASSPATH}:`dirname ${HIVE_SITE_XML}` + + # add jars from db connectivity dir - be careful to not point to something like /lib + for f in ${DBROOT}/*.jar; do + AUX_CLASSPATH=${AUX_CLASSPATH}:$f + done + + # add jars from lib dir + for f in ${HCAT_PREFIX}/share/hcatalog/lib/*.jar ; do + AUX_CLASSPATH=${AUX_CLASSPATH}:$f + done + + # echo AUX_CLASSPATH = ${AUX_CLASSPATH} + export AUX_CLASSPATH=${AUX_CLASSPATH} + + export HADOOP_HOME=$HADOOP_HOME + #export HADOOP_OPTS="-Dlog4j.configuration=file://${HCAT_PREFIX}/conf/log4j.properties" + export HADOOP_OPTS="${HADOOP_OPTS} -server -XX:+UseConcMarkSweepGC -XX:ErrorFile=${HCAT_LOG_DIR}/hcat_err_pid%p.log -Xloggc:${HCAT_LOG_DIR}/hcat_gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps" + export HADOOP_HEAPSIZE=2048 # 8G is better if you have it + export METASTORE_PORT=${METASTORE_PORT} + nohup ${HCAT_PREFIX}/bin/hive --service metastore >${HCAT_LOG_DIR}/hcat.out 2>${HCAT_LOG_DIR}/hcat.err & + + PID=$! + + if [ "${PID}x" == "x" ] ; then # we failed right off + echo "Metastore startup failed, see ${HCAT_LOG_DIR}/hcat.err" + exit 1 + fi + + echo Started metastore server init, testing if initialized correctly... + sleep $SLEEP_TIME_AFTER_START + + if ps -p $PID > /dev/null + then + echo $PID > $PID_FILE + echo "Metastore initialized successfully on port[${METASTORE_PORT}]." + else + echo "Metastore startup failed, see ${HCAT_LOG_DIR}/hcat.err" + exit 1 + fi +} + +function stop_hcat() { + SLEEP_TIME_AFTER_KILL=30 + + PID_FILE=${HCAT_PID_DIR}/hcat.pid + echo looking for $PID_FILE + + # check if service is already running, if so exit + if [ -s "$PID_FILE" ] ; then + PID=`cat $PID_FILE` + echo "Found metastore server process $PID, killing..." + kill $PID + sleep $SLEEP_TIME_AFTER_KILL + + # if process is still around, use kill -9 + if ps -p $PID > /dev/null ; then + echo "Initial kill failed, getting serious now..." + kill -9 $PID + fi + if ps -p $PID > /dev/null ; then + echo "Wow, even kill -9 failed, giving up; sorry" + exit 1 + else + rm -rf $PID_FILE + echo "Successfully shutdown metastore" + fi + fi +} + +if [ $# = 0 ]; then + print_usage + exit +fi + +COMMAND=$1 +case $COMMAND in + start) + start_hcat + ;; + stop) + stop_hcat + ;; +esac +