diff --git src/assembly/binary-layout.xml src/assembly/binary-layout.xml new file mode 100644 index 0000000..1e2e37e --- /dev/null +++ src/assembly/binary-layout.xml @@ -0,0 +1,90 @@ + + + + + binary + + tar.gz + + + + + ${basedir}/*.txt + + share/doc/hbase + + + conf + etc/hbase + + + bin + 755 + + + src/main/ruby + share/hbase/lib/ruby + + + target + share/hbase + + hbase-${project.version}.jar + hbase-${project.version}-tests.jar + + + + target/hbase-webapps + share/hbase/hbase-webapps + + + target/site + share/doc/hbase + + + src/packages/templates/conf + share/hbase/templates/conf + + + src/packages + sbin + 755 + + hbase-setup-conf.sh + update-hbase-env.sh + + + + + + share/hbase/lib + false + runtime + + org.apache.hbase:hbase + + + + diff --git src/assembly/deb-layout.xml src/assembly/deb-layout.xml new file mode 100644 index 0000000..c31c603 --- /dev/null +++ src/assembly/deb-layout.xml @@ -0,0 +1,95 @@ + + + + + deb + + dir + + + + + ${basedir}/*.txt + + share/doc/hbase + + + conf + etc/hbase + + + bin + 755 + + + src/main/ruby + share/hbase/lib/ruby + + + target + share/hbase + + hbase-${project.version}.jar + hbase-${project.version}-tests.jar + + + + target/hbase-webapps + share/hbase/hbase-webapps + + + target/site + share/doc/hbase + + + src/packages/templates/conf + share/hbase/templates/conf + + + src/packages + sbin + 755 + + hbase-setup-conf.sh + update-hbase-env.sh + + + + src/packages/deb/init.d + sbin + 755 + + + + + share/hbase/lib + false + runtime + + org.apache.hbase:hbase + + + + diff --git src/assembly/rpm-layout.xml src/assembly/rpm-layout.xml new file mode 100644 index 0000000..314d919 --- /dev/null +++ src/assembly/rpm-layout.xml @@ -0,0 +1,95 @@ + + + + + rpm + + tar.gz + + + + + ${basedir}/*.txt + + share/doc/hbase + + + conf + etc/hbase + + + bin + 755 + + + src/main/ruby + share/hbase/lib/ruby + + + target + share/hbase + + hbase-${project.version}.jar + hbase-${project.version}-tests.jar + + + + target/hbase-webapps + share/hbase/hbase-webapps + + + target/site + share/doc/hbase + + + src/packages/templates/conf + share/hbase/templates/conf + + + src/packages + sbin + 755 + + hbase-setup-conf.sh + update-hbase-env.sh + + + + src/packages/rpm/init.d + sbin + 755 + + + + + share/hbase/lib + false + runtime + + org.apache.hbase:hbase + + + + diff --git src/packages/conf-pseudo/hbase-site.xml src/packages/conf-pseudo/hbase-site.xml deleted file mode 100644 index b4e8de6..0000000 --- src/packages/conf-pseudo/hbase-site.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - hbase.rootdir - hdfs://localhost:9000/hbase - - - hbase.cluster.distributed - true - The mode the cluster will be in. Possible values are - false: standalone and pseudo-distributed setups with managed Zookeeper - true: fully-distributed with unmanaged Zookeeper Quorum (see hbase-env.sh) - - - - hbase.zookeeper.property.clientPort - 2181 - Property from ZooKeeper's config zoo.cfg. - The port at which the clients will connect. - - - - dfs.replication - 1 - The replication count for HLog and HFile storage. Should not be greater than HDFS datanode count. - - - - hbase.zookeeper.quorum - localhost - Comma separated list of servers in the ZooKeeper Quorum. - For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com". - By default this is set to localhost for local and pseudo-distributed modes - of operation. For a fully-distributed setup, this should be set to a full - list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh - this is the list of servers which we will start/stop ZooKeeper on. - - - - hbase.zookeeper.property.dataDir - /var/lib/zookeeper/data - Property from ZooKeeper's config zoo.cfg. - The directory where the snapshot is stored. - - - diff --git bin/hbase bin/hbase index 3e569ba..251bce0 100755 --- bin/hbase +++ bin/hbase @@ -145,6 +145,25 @@ add_maven_test_classes_to_classpath() { fi } +add_system_classpath() { + # ZooKeeper jar files + f="${HBASE_HOME}/share/zookeeper" + if [ -d "${f}" ]; then + CLASSPATH=${CLASSPATH}:${f}/zookeeper*.jar + fi + # Hadoop jar files + f="${HBASE_HOME}/share/hadoop" + if [ -d "${f}" ]; then + CLASSPATH=${CLASSPATH}:${f}/* + fi + # HBase jar files + f="${HBASE_HOME}/share/hbase" + if [ -d "${f}" ]; then + CLASSPATH=${CLASSPATH}:${f}/lib/* + CLASSPATH=${CLASSPATH}:${f}/* + fi +} + # Add maven target directory if $in_dev_env; then add_maven_deps_to_classpath @@ -152,6 +171,10 @@ if $in_dev_env; then add_maven_test_classes_to_classpath fi +if [ -d "$HBASE_HOME/share/hbase" ]; then + add_system_classpath +fi + # For releases, add hbase & webapps to CLASSPATH # Webapps must come first else it messes up Jetty if [ -d "$HBASE_HOME/hbase-webapps" ]; then diff --git bin/hirb.rb bin/hirb.rb index d6892b5..691cbfe 100644 --- bin/hirb.rb +++ bin/hirb.rb @@ -41,6 +41,8 @@ require 'irb/completion' # to the ruby load path so I can load up my HBase ruby modules if File.exists?(File.join(File.dirname(__FILE__), "..", "lib", "ruby", "hbase.rb")) $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "lib", "ruby") +elsif File.exists?(File.join(File.dirname(__FILE__), "..", "share", "hbase", "lib", "ruby", "hbase.rb")) + $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "share", "hbase", "lib", "ruby") else $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "src", "main", "ruby") end diff --git pom.xml pom.xml index 1555e90..bb027ec 100644 --- pom.xml +++ pom.xml @@ -469,13 +469,6 @@ maven-assembly-plugin - - gnu - false - - src/assembly/all.xml - - tarball @@ -483,6 +476,13 @@ single + + gnu + false + + src/assembly/all.xml + + @@ -755,7 +755,7 @@ /var/run/hbase 1 - 0.91.0 + 0.92.0 ${project.artifactId}-${project.version} @@ -1077,6 +1077,26 @@ + maven-assembly-plugin + + + binary-tarball-rpm + package + + single + + + ${project.artifactId}-${project.version}-rpm + gnu + false + + src/assembly/rpm-layout.xml + + + + + + maven-antrun-plugin 1.6 @@ -1087,7 +1107,6 @@ - @@ -1105,6 +1124,26 @@ + maven-assembly-plugin + + + binary-tarball-deb + package + + single + + + ${project.artifactId}-${project.version}-deb + gnu + false + + src/assembly/deb-layout.xml + + + + + + maven-antrun-plugin 1.6 @@ -1116,7 +1155,6 @@ - @@ -1136,6 +1174,33 @@ + + binary + + + + maven-assembly-plugin + + + binary-tarball + package + + single + + + ${project.artifactId}-${project.version}-binary + gnu + false + + src/assembly/binary-layout.xml + + + + + + + + os.linux diff --git src/assembly/all.xml src/assembly/all.xml index 9070a32..85a8815 100644 --- src/assembly/all.xml +++ src/assembly/all.xml @@ -69,19 +69,6 @@ target/site docs - - src/packages/templates/conf - share/hbase/templates/conf - - - src/packages - sbin - 755 - - hbase-setup-conf.sh - update-hbase-env.sh - - diff --git src/docbkx/getting_started.xml src/docbkx/getting_started.xml index 0d0df0f..d5a9241 100644 --- src/docbkx/getting_started.xml +++ src/docbkx/getting_started.xml @@ -74,6 +74,14 @@ $ cd hbase- design to run as root user to setup directory and file permissions. This script is designed to configure HBase to work with Hadoop. For running standalone HBase, skip to section. + + + Setup application directory for HBase to write on HDFS. + + $ sudo hadoop-setup-application.sh --apps hbase + + + Setup HBase configuration $ sudo hbase-setup-conf.sh --hadoop-home=/usr \ --hadoop-conf=/etc/hadoop \ diff --git src/packages/build.xml src/packages/build.xml index e17295f..58df5e1 100644 --- src/packages/build.xml +++ src/packages/build.xml @@ -36,51 +36,24 @@ - - - - - - + + + + - - - - - - - + + + + + - - - - + + - - - - - - - - - - - - - - - - - - - - - - - + + + @@ -91,13 +64,13 @@ - + - + - + @@ -116,34 +89,4 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git src/packages/deb/hbase.control/postinst src/packages/deb/hbase.control/postinst index a77fe4f..e0a8fd5 100644 --- src/packages/deb/hbase.control/postinst +++ src/packages/deb/hbase.control/postinst @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -bash /usr/share/hbase/sbin/update-hbase-env.sh \ +bash /usr/sbin/update-hbase-env.sh \ --prefix=/usr \ --bin-dir=/usr/bin \ --conf-dir=/etc/hbase \ diff --git src/packages/deb/hbase.control/preinst src/packages/deb/hbase.control/preinst index 497bbab..ec229b7 100644 --- src/packages/deb/hbase.control/preinst +++ src/packages/deb/hbase.control/preinst @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -r hadoop +getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -g 123 -r hadoop -/usr/sbin/useradd --comment "HBase" --shell /bin/bash -M -r --groups hadoop --home /usr/share/hbase hbase 2> /dev/null || : +/usr/sbin/useradd -u 203 --comment "HBase" --shell /bin/bash -M -r -g hadoop --home /usr/share/hbase hbase 2> /dev/null || : diff --git src/packages/deb/hbase.control/prerm src/packages/deb/hbase.control/prerm index 55486e6..ba237e9 100644 --- src/packages/deb/hbase.control/prerm +++ src/packages/deb/hbase.control/prerm @@ -17,7 +17,7 @@ /etc/init.d/hbase-master stop 2>/dev/null >/dev/null /etc/init.d/hbase-regionserver stop 2>/dev/null >/dev/null -bash /usr/share/hbase/sbin/update-hbase-env.sh \ +bash /usr/sbin/update-hbase-env.sh \ --prefix=/usr \ --bin-dir=/usr/bin \ --conf-dir=/etc/hbase \ diff --git src/packages/deb/init.d/hbase-master src/packages/deb/init.d/hbase-master index 55eea9b..19e66cb 100644 --- src/packages/deb/init.d/hbase-master +++ src/packages/deb/init.d/hbase-master @@ -73,7 +73,7 @@ case "$1" in check_privsep_dir check_for_no_start log_daemon_msg "Starting Apache HBase Master server" "hbase-master" - if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-master.pid -c hbase -x ${HBASE_HOME}/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start master; then + if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-master.pid -c hbase -x /usr/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start master; then log_end_msg 0 else log_end_msg 1 @@ -93,7 +93,7 @@ case "$1" in log_daemon_msg "Restarting Apache HBase Master server" "hbase-master" start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HBASE_PID_DIR}/hbase-hbase-master.pid check_for_no_start log_end_msg - if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-master.pid -c hbase -x ${HBASE_HOME}/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start master; then + if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-master.pid -c hbase -x /usr/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start master; then log_end_msg 0 else log_end_msg 1 diff --git src/packages/deb/init.d/hbase-regionserver src/packages/deb/init.d/hbase-regionserver index 0aaa507..71d30ef 100644 --- src/packages/deb/init.d/hbase-regionserver +++ src/packages/deb/init.d/hbase-regionserver @@ -73,7 +73,7 @@ case "$1" in check_privsep_dir check_for_no_start log_daemon_msg "Starting Apache HBase Region Server server" "hbase-regionserver" - if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-regionserver.pid -c hbase -x ${HBASE_HOME}/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start regionserver; then + if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-regionserver.pid -c hbase -x /usr/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start regionserver; then log_end_msg 0 else log_end_msg 1 @@ -93,7 +93,7 @@ case "$1" in log_daemon_msg "Restarting Apache HBase Region Server server" "hbase-regionserver" start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HBASE_PID_DIR}/hbase-hbase-regionserver.pid check_for_no_start log_end_msg - if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-regionserver.pid -c hbase -x ${HBASE_HOME}/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start regionserver; then + if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-regionserver.pid -c hbase -x /usr/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start regionserver; then log_end_msg 0 else log_end_msg 1 @@ -111,7 +111,7 @@ case "$1" in 0) # old daemon stopped check_for_no_start log_end_msg - if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-regionserver.pid -c hbase -x ${HBASE_HOME}/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start regionserver; then + if start-stop-daemon --start --quiet --oknodo --pidfile ${HBASE_PID_DIR}/hbase-hbase-regionserver.pid -c hbase -x /usr/bin/hbase-daemon.sh -- --config ${HBASE_CONF_DIR} start regionserver; then log_end_msg 0 else log_end_msg 1 diff --git src/packages/hbase-setup-conf.sh src/packages/hbase-setup-conf.sh index 112dfb8..5bba846 100644 --- src/packages/hbase-setup-conf.sh +++ src/packages/hbase-setup-conf.sh @@ -176,8 +176,8 @@ HADOOP_NAMENODE=${HADOOP_NAMENODE:-localhost} HADOOP_REPLICATION=${HADOOP_REPLICATION:-3} HBASE_HOME=${HBASE_HOME:-/usr} HBASE_CONF_DIR=${HBASE_CONF_DIR:-/etc/hbase} -HBASE_LOG_DIR=${HBASE_LOG_DIR:-$HBASE_HOME/var/log} -HBASE_PID_DIR=${HBASE_PID_DIR:-$HBASE_HOME/var/run} +HBASE_LOG_DIR=${HBASE_LOG_DIR:-/var/log/hbase} +HBASE_PID_DIR=${HBASE_PID_DIR:-/var/run/hbase} HBASE_USER=${HBASE_USER:-hbase} KEYTAB_DIR=${KEYTAB_DIR:-/etc/security/keytabs} REGION_SERVERS=${REGION_SERVERS:-localhost} @@ -255,256 +255,3 @@ chown ${HBASE_USER} ${ZOOKEEPER_SNAPSHOT} chmod 755 ${HBASE_CONF_DIR}/hbase-env.sh echo "HBase Configuration setup is completed." -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -#if [ -e "$bin"/../libexec/hbase-config.sh ]; then -# . "$bin"/../libexec/hbase-config.sh -#elif [ -e "$bin"/hbase-config.sh ]; then -# . "$bin"/hbase-config.sh -#else -# echo "hbase-config.sh is not found." -# exit 1 -#fi - -usage() { - echo " -usage: $0 - - Optional parameters: - --hadoop-conf=/usr/etc/hadoop Set Hadoop configuration directory location - --hadoop-home=/usr Set Hadoop directory location - --hadoop-namenode=localhost Set Hadoop namenode hostname - --hadoop-replication=3 Set HDFS replication - --hbase-home=/usr Set HBase directory location - --hbase-conf=/etc/hbase Set HBase configuration directory location - --hbase-log=/var/log/hbase Set HBase log directory location - --hbase-pid=/var/run/hbase Set HBase pid directory location - --hbase-user=hbase Set HBase user - --java-home=/usr/java/default Set JAVA_HOME directory location - --kerberos-realm=KERBEROS.EXAMPLE.COM Set Kerberos realm - --kerberos-principal-id=_HOST Set Kerberos principal ID - --keytab-dir=/etc/security/keytabs Set keytab directory - --regionservers=localhost Set regionservers hostnames - --zookeeper-home=/usr Set ZooKeeper directory location - --zookeeper-quorum=localhost Set ZooKeeper Quorum - --zookeeper-snapshot=/var/lib/zookeeper Set ZooKeeper snapshot location - " - exit 1 -} - -template_generator() { - REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})' - cat $1 | - while read line ; do - while [[ "$line" =~ $REGEX ]] ; do - LHS=${BASH_REMATCH[1]} - RHS="$(eval echo "\"$LHS\"")" - line=${line//$LHS/$RHS} - done - echo $line >> $2 - done -} - -OPTS=$(getopt \ - -n $0 \ - -o '' \ - -l 'hadoop-conf:' \ - -l 'hadoop-home:' \ - -l 'hadoop-namenode:' \ - -l 'hadoop-replication:' \ - -l 'hbase-conf:' \ - -l 'hbase-home:' \ - -l 'hbase-log:' \ - -l 'hbase-pid:' \ - -l 'hbase-user:' \ - -l 'java-home:' \ - -l 'keytab-dir:' \ - -l 'kerberos-principal-id:' \ - -l 'kerberos-realm:' \ - -l 'regionservers:' \ - -l 'zookeeper-home:' \ - -l 'zookeeper-quorum:' \ - -l 'zookeeper-snapshot:' \ - -o 'h' \ - -- "$@") - -if [ $? != 0 ] ; then - usage -fi - -eval set -- "${OPTS}" -while true ; do - case "$1" in - --hadoop-conf) - HADOOP_CONF_DIR=$2 - shift 2 - ;; - --hadoop-home) - HADOOP_HOME=$2 - shift 2 - ;; - --hadoop-namenode) - HADOOP_NAMENODE=$2 - shift 2 - ;; - --hadoop-replication) - HADOOP_REPLICATION=$2 - shift 2 - ;; - --hbase-conf) - HBASE_CONF_DIR=$2 - shift 2 - ;; - --hbase-home) - HBASE_HOME=$2 - shift 2 - ;; - --hbase-log) - HBASE_LOG_DIR=$2 - shift 2 - ;; - --hbase-pid) - HBASE_PID_DIR=$2 - shift 2 - ;; - --hbase-user) - HBASE_USER=$2 - shift 2 - ;; - --java-home) - JAVA_HOME=$2 - shift 2 - ;; - --kerberos-principal-id) - KERBEROS_PRINCIPAL_ID=$2 - shift 2 - ;; - --kerberos-realm) - KERBEROS_REALM=$2 - shift 2 - ;; - --keytab-dir) - KEYTAB_DIR=$2; shift 2 - ;; - --regionservers) - REGION_SERVERS=$2 - shift 2 - ;; - --zookeeper-home) - ZOOKEEPER_HOME=$2 - shift 2 - ;; - --zookeeper-quorum) - ZOOKEEPER_QUORUM=$2 - shift 2 - ;; - --zookeeper-snapshot) - ZOOKEEPER_SNAPSHOT=$2 - shift 2 - ;; - --) - shift ; break - ;; - *) - echo "Unknown option: $1" - usage - ;; - esac -done - -JAVA_HOME=${JAVA_HOME:-/usr/java/default} -HADOOP_HOME=${HADOOP_HOME:-/usr} -HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop} -HADOOP_NAMENODE=${HADOOP_NAMENODE:-localhost} -HADOOP_REPLICATION=${HADOOP_REPLICATION:-3} -HBASE_HOME=${HBASE_HOME:-/usr} -HBASE_CONF_DIR=${HBASE_CONF_DIR:-/etc/hbase} -HBASE_USER=${HBASE_USER:-hbase} -KEYTAB_DIR=${KEYTAB_DIR:-/etc/security/keytabs} -KERBEROS_PRINCIPAL_ID=${KERBEROS_PRINCIPAL_ID:-_HOST} -REGION_SERVERS=${REGION_SERVERS:-localhost} -REGION_SERVERS=`echo ${REGION_SERVERS} | sed -e 's/,/ /g'` -ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/usr} -ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-localhost} -ZOOKEEPER_SNAPSHOT=${ZOOKEEPER_SNAPSHOT:-/var/lib/zookeeper} -if [ "${KERBEROS_REALM}" != "" ]; then - HBASE_M_K_PRINCIPAL="${KERBEROS_PRINCIPAL_ID}@${KERBEROS_REALM}" - HBASE_R_K_PRINCIPAL="${KERBEROS_PRINCIPAL_ID}@${KERBEROS_REALM}" - if [ "${KERBEROS_PRINCIPAL_ID}" = "_HOST" ]; then - HBASE_M_K_FILE="${KEYTAB_DIR}/hm.service.keytab" - HBASE_R_K_FILE="${KEYTAB_DIR}/rs.service.keytab" - else - HBASE_M_K_FILE="${KEYTAB_DIR}/${KERBEROS_PRINCIPAL_ID}.keytab" - HBASE_R_K_FILE="${KEYTAB_DIR}/${KERBEROS_PRINCIPAL_ID}.keytab" - fi -else - HBASE_M_K_PRINCIPAL="" - HBASE_R_K_PRINCIPAL="" - HBASE_M_K_FILE="" - HBASE_R_K_FILE="" -fi - -rm -f ${HBASE_HOME}/lib/hadoop-core*.jar 2>/dev/null - -HADOOP_JAR=`ls ${HADOOP_HOME}/hadoop-core*.jar | head -n1` -COMMONCONF_JAR=`ls ${HADOOP_HOME}/lib/commons-configuration*.jar 2>/dev/null | head -n1` -if [ "x${HADOOP_JAR}" != "x" ]; then - HBASE_CLASSPATH=${HADOOP_JAR}:${COMMONCONF_JAR} -else - COMMON_JAR=`ls ${HADOOP_HOME}/hadoop-common*.jar 2>/dev/null | head -n1` - HDFS_JAR=`ls ${HADOOP_HOME}/hadoop-hdfs*.jar 2>/dev/null | head -n1` - MAPREDUCE_JAR=`ls ${HADOOP_HOME}/hadoop-mapred*.jar 2>/dev/null | head -n1` - COMMONCONF_JAR=`ls ${HADOOP_HOME}/lib/commons-configuration*.jar 2>/dev/null | head -n1` - - if [ "x${COMMON_JAR}" != "x" ]; then - HBASE_CLASSPATH=${COMMON_JAR}:${HDFS_JAR}:${MAPREDUCE_JAR}:${COMMONCONF_JAR} - else - echo "Can not find Hadoop jar files." - exit 1 - fi -fi - -if [ -e ${HBASE_CONF_DIR}/hbase-env.sh ]; then - rm -f ${HBASE_CONF_DIR}/hbase-env.sh -fi - -if [ -e ${HBASE_CONF_DIR}/hbase-site.xml ]; then - rm -f ${HBASE_CONF_DIR}/hbase-site.xml -fi - -template_generator ${HBASE_HOME}/share/hbase/templates/conf/hbase-env.sh ${HBASE_CONF_DIR}/hbase-env.sh -template_generator ${HBASE_HOME}/share/hbase/templates/conf/hbase-site.xml ${HBASE_CONF_DIR}/hbase-site.xml - -rm -f ${HBASE_CONF_DIR}/regionservers -for host in ${REGION_SERVERS} -do - echo $host >> ${HBASE_CONF_DIR}/regionservers -done - -mkdir -p ${HBASE_LOG_DIR} -mkdir -p ${HBASE_PID_DIR} -mkdir -p ${ZOOKEEPER_SNAPSHOT} -chown ${HBASE_USER} ${HBASE_LOG_DIR} -chown ${HBASE_USER} ${HBASE_PID_DIR} -chown ${HBASE_USER} ${ZOOKEEPER_SNAPSHOT} - -chmod 755 ${HBASE_CONF_DIR}/hbase-env.sh -echo "HBase Configuration setup is completed." diff --git src/packages/rpm/init.d/hbase-master src/packages/rpm/init.d/hbase-master index 67752da..36df443 100644 --- src/packages/rpm/init.d/hbase-master +++ src/packages/rpm/init.d/hbase-master @@ -22,7 +22,8 @@ # description: HBase master source /etc/rc.d/init.d/functions -source /etc/default/hbase-env.sh +source /etc/hbase/hbase-env.sh +HBASE_HOME=/usr RETVAL=0 PIDFILE="${HBASE_PID_DIR}/hbase-hbase-master.pid" diff --git src/packages/rpm/init.d/hbase-regionserver src/packages/rpm/init.d/hbase-regionserver index 186ea21..f23a55a 100644 --- src/packages/rpm/init.d/hbase-regionserver +++ src/packages/rpm/init.d/hbase-regionserver @@ -22,7 +22,8 @@ # description: HBase region server source /etc/rc.d/init.d/functions -source /etc/default/hbase-env.sh +source /etc/hbase/hbase-env.sh +HBASE_HOME=/usr RETVAL=0 PIDFILE="${HBASE_PID_DIR}/hbase-hbase-regionserver.pid" diff --git src/packages/rpm/spec/hbase.spec src/packages/rpm/spec/hbase.spec index 7d4207a..5b0a6a5 100644 --- src/packages/rpm/spec/hbase.spec +++ src/packages/rpm/spec/hbase.spec @@ -44,7 +44,7 @@ %define _final_name @final.name@ %define debug_package %{nil} -Summary: Default HBase configuration templates +Summary: Apache HBase is the Hadoop database. License: Apache License, Version 2.0 URL: http://hbase.apache.org/ Vendor: Apache Software Foundation @@ -63,7 +63,13 @@ AutoReqProv: no Provides: hbase %description -Installation of this RPM will setup your machine to run in pseudo-distributed mode where each HBase daemon runs in a separate Java process. +HBase is the Hadoop database. Use it when you need random, realtime read/write access +to your Big Data. This project's goal is the hosting of very large tables -- billions +of rows X millions of columns -- atop clusters of commodity hardware. HBase is an +open-source, distributed, versioned, column-oriented store modeled after Google' +Bigtable: A Distributed Storage System for Structured by Chang et al. Just as Bigtable +leverages the distributed data storage provided by the Google File System, HBase +provides Bigtable-like capabilities on top of Hadoop. %prep %setup -n %{_final_name} @@ -85,32 +91,24 @@ if [ -d ${RPM_BUILD_DIR}%{_pid_dir} ]; then rm -rf ${RPM_BUILD_DIR}%{_pid_dir} fi -mkdir -p ${RPM_BUILD_DIR}%{_conf_dir} -mkdir -p ${RPM_BUILD_DIR}%{_bin_dir} -mkdir -p ${RPM_BUILD_DIR}%{_include_dir} -mkdir -p ${RPM_BUILD_DIR}%{_lib_dir} -mkdir -p ${RPM_BUILD_DIR}%{_libexec_dir} +mkdir -p ${RPM_BUILD_DIR}%{_prefix} mkdir -p ${RPM_BUILD_DIR}%{_log_dir} mkdir -p ${RPM_BUILD_DIR}%{_conf_dir} -mkdir -p ${RPM_BUILD_DIR}%{_man_dir} mkdir -p ${RPM_BUILD_DIR}%{_pid_dir} -mkdir -p ${RPM_BUILD_DIR}%{_sbin_dir} -mkdir -p ${RPM_BUILD_DIR}%{_share_dir} -mkdir -p ${RPM_BUILD_DIR}%{_src_dir} - -cp ${RPM_BUILD_DIR}/%{_final_name}/src/packages/update-hbase-env.sh ${RPM_BUILD_DIR}/%{_final_name}/sbin/update-hbase-env.sh -cp ${RPM_BUILD_DIR}/%{_final_name}/src/packages/rpm/init.d/hbase-master ${RPM_BUILD_DIR}/%{_final_name}/sbin/hbase-master -cp ${RPM_BUILD_DIR}/%{_final_name}/src/packages/rpm/init.d/hbase-regionserver ${RPM_BUILD_DIR}/%{_final_name}/sbin/hbase-regionserver -chmod 0755 ${RPM_BUILD_DIR}/%{_final_name}/sbin/* -rm -f ${RPM_BUILD_DIR}/%{_final_name}/lib/hadoop-core* -rm -f ${RPM_BUILD_DIR}/%{_final_name}/lib/zookeeper* -mv -f ${RPM_BUILD_DIR}/%{_final_name}/conf/* ${RPM_BUILD_DIR}%{_conf_dir} -mv -f ${RPM_BUILD_DIR}/%{_final_name}/* ${RPM_BUILD_DIR}%{_share_dir} +mkdir -p ${RPM_BUILD_DIR}/etc/rc.d/init.d + +mv -f ${RPM_BUILD_DIR}/%{_final_name}/sbin/hbase-master ${RPM_BUILD_DIR}/etc/rc.d/init.d/hbase-master +mv -f ${RPM_BUILD_DIR}/%{_final_name}/sbin/hbase-regionserver ${RPM_BUILD_DIR}/etc/rc.d/init.d/hbase-regionserver +mv -f ${RPM_BUILD_DIR}/%{_final_name}/etc/hbase/* ${RPM_BUILD_DIR}%{_conf_dir} +mv -f ${RPM_BUILD_DIR}/%{_final_name}/* ${RPM_BUILD_DIR}%{_prefix} +chmod 0755 ${RPM_BUILD_DIR}/etc/rc.d/init.d/* +chmod 0755 ${RPM_BUILD_DIR}/%{_prefix}/bin/* +chmod 0755 ${RPM_BUILD_DIR}/%{_prefix}/sbin/* rm -rf ${RPM_BUILD_DIR}/%{_final_name} %preun -${RPM_INSTALL_PREFIX0}/share/hbase/sbin/update-hbase-env.sh \ +${RPM_INSTALL_PREFIX0}/sbin/update-hbase-env.sh \ --prefix=${RPM_INSTALL_PREFIX0} \ --bin-dir=${RPM_INSTALL_PREFIX0}/bin \ --conf-dir=${RPM_INSTALL_PREFIX1} \ @@ -119,12 +117,12 @@ ${RPM_INSTALL_PREFIX0}/share/hbase/sbin/update-hbase-env.sh \ --uninstall %pre -getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -r hadoop +getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -g 123 -r hadoop -/usr/sbin/useradd --comment "HBase" --shell /bin/bash -M -r --groups hadoop --home %{_share_dir} hbase 2> /dev/null || : +/usr/sbin/useradd --comment "HBase" -u 203 --shell /bin/bash -M -r --groups hadoop --home %{_share_dir} hbase 2> /dev/null || : %post -${RPM_INSTALL_PREFIX0}/share/hbase/sbin/update-hbase-env.sh \ +${RPM_INSTALL_PREFIX0}/sbin/update-hbase-env.sh \ --prefix=${RPM_INSTALL_PREFIX0} \ --bin-dir=${RPM_INSTALL_PREFIX0}/bin \ --conf-dir=${RPM_INSTALL_PREFIX1} \ @@ -134,4 +132,5 @@ ${RPM_INSTALL_PREFIX0}/share/hbase/sbin/update-hbase-env.sh \ %files %defattr(-,root,root) %{_prefix} -%config %{_conf_dir} +%config(noreplace) %{_conf_dir} +%attr(0755,root,root) /etc/rc.d/init.d diff --git src/packages/templates/conf/hbase-env.sh src/packages/templates/conf/hbase-env.sh index 6c7d710..5a5b1bf 100644 --- src/packages/templates/conf/hbase-env.sh +++ src/packages/templates/conf/hbase-env.sh @@ -50,83 +50,7 @@ export HBASE_OPTS="-ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode" # export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" # File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. -export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers - -# Extra ssh options. Empty by default. -# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" - -# Where log files are stored. $HBASE_HOME/logs by default. -export HBASE_LOG_DIR=${HBASE_LOG_DIR} - -# A string representing this instance of hbase. $USER by default. -# export HBASE_IDENT_STRING=$USER - -# The scheduling priority for daemon processes. See 'man nice'. -# export HBASE_NICENESS=10 - -# The directory where pid files are stored. /tmp by default. -export HBASE_PID_DIR=${HBASE_PID_DIR} - -# Seconds to sleep between slave commands. Unset by default. This -# can be useful in large clusters, where, e.g., slave rsyncs can -# otherwise arrive faster than the master can service them. -# export HBASE_SLAVE_SLEEP=0.1 - -# Tell HBase whether it should manage it's own instance of Zookeeper or not. -export HBASE_MANAGES_ZK=true -# -# -# Copyright 2007 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Set environment variables here. - -# The java implementation to use. Java 1.6 required. -export JAVA_HOME=${JAVA_HOME} - -# Extra Java CLASSPATH elements. Optional. -export HBASE_CLASSPATH=${HADOOP_CONF_DIR}:${HBASE_CLASSPATH} - -# The maximum amount of heap to use, in MB. Default is 1000. -# export HBASE_HEAPSIZE=1000 - -# Extra Java runtime options. -# Below are what we set by default. May only work with SUN JVM. -# For more on why as well as other possible settings, -# see http://wiki.apache.org/hadoop/PerformanceTuning -export HBASE_OPTS="-ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode" - -# Uncomment below to enable java garbage collection logging. -# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" - -# Uncomment and adjust to enable JMX exporting -# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. -# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# -# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" -# export HBASE_MASTER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101 -javaagent:lib/HelloWorldAgent.jar" -# export HBASE_REGIONSERVER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" -# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" -# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" - -# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. -export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers +export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers # Extra ssh options. Empty by default. # export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" diff --git src/packages/templates/conf/hbase-site.xml src/packages/templates/conf/hbase-site.xml index c6ae59d..7696f75 100644 --- src/packages/templates/conf/hbase-site.xml +++ src/packages/templates/conf/hbase-site.xml @@ -22,97 +22,7 @@ hbase.rootdir - hdfs://${HADOOP_NAMENODE}:8020/hbase - The directory shared by RegionServers. - - - - dfs.replication - ${HADOOP_REPLICATION} - The replication count for HLog and HFile storage. Should not be greater than HDFS datanode count. - - - - hbase.cluster.distributed - true - The mode the cluster will be in. Possible values are - false: standalone and pseudo-distributed setups with managed Zookeeper - true: fully-distributed with unmanaged Zookeeper Quorum (see hbase-env.sh) - - - - hbase.zookeeper.property.clientPort - 2181 - Property from ZooKeeper's config zoo.cfg. - The port at which the clients will connect. - - - - hbase.zookeeper.quorum - ${ZOOKEEPER_QUORUM} - Comma separated list of servers in the ZooKeeper Quorum. - For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com". - By default this is set to localhost for local and pseudo-distributed modes - of operation. For a fully-distributed setup, this should be set to a full - list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh - this is the list of servers which we will start/stop ZooKeeper on. - - - - hbase.zookeeper.property.dataDir - ${ZOOKEEPER_SNAPSHOT} - Property from ZooKeeper's config zoo.cfg. - The directory where the snapshot is stored. - - - - hbase.master.keytab.file - ${HBASE_M_K_FILE} - - - - hbase.master.kerberos.principal - ${HBASE_M_K_PRINCIPAL} - - - - hbase.regionserver.keytab.file - ${HBASE_R_K_FILE} - - - - hbase.regionserver.kerberos.principal - ${HBASE_R_K_PRINCIPAL} - - - - - - - - - hbase.rootdir - hdfs://${HADOOP_NAMENODE}:8020/hbase + hdfs://${HADOOP_NAMENODE}:8020/apps/hbase/data The directory shared by RegionServers. diff --git src/packages/update-hbase-env.sh src/packages/update-hbase-env.sh index 8fec1f5..f93dead 100644 --- src/packages/update-hbase-env.sh +++ src/packages/update-hbase-env.sh @@ -34,6 +34,19 @@ usage: $0 exit 1 } +template_generator() { + REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})' + cat $1 | + while read line ; do + while [[ "$line" =~ $REGEX ]] ; do + LHS=${BASH_REMATCH[1]} + RHS="$(eval echo "\"$LHS\"")" + line=${line//$LHS/$RHS} + done + echo $line >> $2 + done +} + OPTS=$(getopt \ -n $0 \ -o '' \ @@ -97,7 +110,7 @@ for var in PREFIX; do done ARCH=${ARCH:-i386} -BIN_DIR=${BIN_DIR:-$PREFIX/share/hbase/bin} +BIN_DIR=${BIN_DIR:-$PREFIX/bin} CONF_DIR=${CONF_DIR:-$PREFIX/conf} LIB_DIR=${LIB_DIR:-$PREFIX/lib} LOG_DIR=${LOG_DIR:-$PREFIX/var/log} @@ -113,48 +126,14 @@ fi if [ "${UNINSTALL}" -eq "1" ]; then # Remove symlinks - if [ "${BIN_DIR}" != "${PREFIX}/share/hbase/bin" ]; then - for var in `ls ${PREFIX}/share/hbase/bin`; do - rm -f ${BIN_DIR}/${var} - done - fi if [ -f /etc/default/hbase-env.sh ]; then rm -f /etc/default/hbase-env.sh fi - if [ "${CONF_DIR}" != "${PREFIX}/share/hbase/conf" ]; then - rm -f ${PREFIX}/share/hbase/conf - fi - - rm -f ${PREFIX}/share/hbase/sbin/hbase-master - rm -f ${PREFIX}/share/hbase/sbin/hbase-regionserver - rm -f /etc/init.d/hbase-master - rm -f /etc/init.d/hbase-regionserver - else # Create symlinks - if [ "${BIN_DIR}" != "${PREFIX}/share/hbase/bin" ]; then - for var in `ls ${PREFIX}/share/hbase/bin`; do - ln -sf ${PREFIX}/share/hbase/bin/${var} ${BIN_DIR}/${var} - done - fi - if [ "${CONF_DIR}" != "${PREFIX}/share/hbase/conf" ]; then - ln -sf ${CONF_DIR} ${PREFIX}/share/hbase/conf - fi - - chmod 755 ${PREFIX}/share/hbase/sbin/* - - ln -sf ${PREFIX}/share/hbase/sbin/hbase-master /etc/init.d/hbase-master - ln -sf ${PREFIX}/share/hbase/sbin/hbase-regionserver /etc/init.d/hbase-regionserver - ln -sf ${CONF_DIR}/hbase-env.sh /etc/default/hbase-env.sh - ln -sf ${CONF_DIR}/hbase-env.sh /etc/profile.d/hbase-env.sh - - if [ -d ${HADOOP_HOME} ]; then - HADOOP_JARS=`ls ${HADOOP_HOME}/*.jar | tr '\n' ':'` - fi - - if [ -d ${ZOOKEEPER_HOME}/share/zookeeper ]; then - ZOOKEEPER_JARS=`ls ${ZOOKEEPER_HOME}/share/zookeeper/*.jar | tr '\n' ':'` + if [ -d /etc/profile.d ]; then + ln -sf ${CONF_DIR}/hbase-env.sh /etc/profile.d/hbase-env.sh fi mkdir -p ${PID_DIR} @@ -163,31 +142,7 @@ else chown hbase ${LOG_DIR} TFILE="/tmp/$(basename $0).$$.tmp" - grep -v "^export HBASE_HOME" ${CONF_DIR}/hbase-env.sh | \ - grep -v "^export HBASE_CONF_DIR" | \ - grep -v "^export HBASE_CLASSPATH" | \ - grep -v "^export HBASE_MANAGES_ZK" | \ - grep -v "^export HBASE_IDENT_STRING" | \ - grep -v "^export HBASE_PID_DIR" | \ - grep -v "^export HBASE_LOG_DIR" | \ - grep -v "^export JAVA_HOME" > ${TFILE} - if [ -z "${JAVA_HOME}" ]; then - if [ -e /etc/lsb-release ]; then - JAVA_HOME=`update-alternatives --config java | grep java | cut -f2 -d':' | cut -f2 -d' ' | sed -e 's/\/bin\/java//'` - else - JAVA_HOME=/usr/java/default - fi - fi - if [ "${JAVA_HOME}xxx" != "xxx" ]; then - echo "export JAVA_HOME=${JAVA_HOME}" >> ${TFILE} - fi - echo "export HBASE_IDENT_STRING=\`whoami\`" >> ${TFILE} - echo "export HBASE_HOME=${PREFIX}/share/hbase" >> ${TFILE} - echo "export HBASE_CONF_DIR=${CONF_DIR}" >> ${TFILE} - echo "export HBASE_CLASSPATH=${CONF_DIR}:${HADOOP_CONF_DIR}:${HADOOP_JARS}:${ZOOKEEPER_JARS}" >> ${TFILE} - echo "export HBASE_MANAGES_ZK=false" >> ${TFILE} - echo "export HBASE_PID_DIR=${PID_DIR}" >> ${TFILE} - echo "export HBASE_LOG_DIR=${LOG_DIR}" >> ${TFILE} + template_generator ${PREFIX}/share/hbase/templates/conf/hbase-env.sh $TFILE cp ${TFILE} ${CONF_DIR}/hbase-env.sh rm -f ${TFILE} fi