From 63fa6c24073ce46f8e3d1c6828a8a37023ad4113 Mon Sep 17 00:00:00 2001 From: Dima Spivak Date: Wed, 3 Sep 2014 16:19:07 -0700 Subject: [PATCH] HBASE-11885 Provide a Dockerfile to easily build and run HBase from source * Add the hbase_docker folder containing the Dockerfile and a basic README. * Add a script called hbase_docker.sh to facilitate building of the Docker image. --- dev-support/hbase_docker.sh | 162 ++++++++++++++++++++++++++++++++++++ dev-support/hbase_docker/Dockerfile | 50 +++++++++++ dev-support/hbase_docker/README.md | 39 +++++++++ 3 files changed, 251 insertions(+) create mode 100755 dev-support/hbase_docker.sh create mode 100644 dev-support/hbase_docker/Dockerfile create mode 100644 dev-support/hbase_docker/README.md diff --git a/dev-support/hbase_docker.sh b/dev-support/hbase_docker.sh new file mode 100755 index 0000000..cc17cf7 --- /dev/null +++ b/dev-support/hbase_docker.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# hbase_docker.sh +# A driver script to build HBase master branch from source and start the HBase +# shell in a Docker container. +# +# Usage: This script automates the build process facilitated by the Dockerfile +# in the hbase_docker folder. In particular, it is assumed that an +# Oracle JDK .tar.gz and an Apache Maven .tar.gz file are both present in +# the same directory as the Dockerfile; this script can download and place +# those tarballs for you. Moreover, due to bugs in Docker, the docker build +# command occasionally fails, but then succeeds upon rerunning; this script +# will rerun the command and attempt to finish a build. Finally, this +# script allows you to specify a desired name for the Docker image being +# created, defaulting to "hbase_docker." For complete usage instructions, +# run this script with the -h or --help option. +# +# Example: To build HBase and start an HBase shell using Oracle JDK 7u67 and +# Apache Maven 3.2.3: +# +# # ./hbase_docker.sh -j http://download.oracle.com/otn-pub/java/jdk/7u67-b01/jdk-7u67-linux-x64.tar.gz \ +# -m http://apache.claz.org/maven/maven-3/3.2.3/binaries/apache-maven-3.2.3-bin.tar.gz + + +# Before doing anything else, make sure Docker is installed. +if ! which docker &> /dev/null; then + cat >&2 << __EOF +Docker must be installed to run hbase_docker. Go to http://www.docker.io +for installation instructions. Exiting... +__EOF + exit 1 +elif ! docker ps &> /dev/null; then + echo "Docker must be run as root or with sudo privileges. Exiting..." >&2 + exit 1 +fi + +# Usage message. +usage() { + SCRIPT=$(basename "${BASH_SOURCE}") + + cat << __EOF + +hbase_docker. A driver script for building HBase and starting the HBase shell +inside of a Docker container. + +Usage: ${SCRIPT} [-j | --jdk ] [-m | --mvn ] [-n | --name ] + ${SCRIPT} -h | --help + + -h | --help Show this screen. + -j | --jdk '' A URL pointing to an x64 .tar.gz file of Oracle's JDK. + Using this argument implies acceptance of the Oracle + Binary Code License Agreement for Java SE. See www.oracle.com + for more details. + -m | --mvn '' A URL pointing to an x64 .tar.gz file of Apache Maven. + -n | --name The name to give to the Docker image created by this script. + If left blank, "hbase_docker" will be used. +__EOF +} + +if ! [ ${#} -le 6 ]; then + usage >&2 + exit 1 +fi + +# Default Docker image name. +IMAGE_NAME=hbase_docker + +while ((${#})); do + case "${1}" in + -h | --help ) + usage; exit 0 ;; + -j | --jdk ) + JDK_URL="${2}"; shift 2 ;; + -m | --mvn ) + MAVEN_URL="${2}"; shift 2 ;; + -n | --name ) + IMAGE_NAME="${2}"; shift 2 ;; + * ) + usage >&2; exit 1 ;; + esac +done + +# The relative file path to the directory containing this script. This allows the +# script to be run from any working directory and still have it place downloaded +# files in the right locations. +SCRIPT_DIRECTORY=$(dirname ${BASH_SOURCE}) + +# If JDK_URL is set, download the JDK into the hbase_docker folder. +if [ -n "${JDK_URL}" ]; then + echo "Downloading Oracle JDK..." + ORACLE_HEADER="Cookie: oraclelicense=accept-securebackup-cookie" + if ! wget -nv -N --header "${ORACLE_HEADER}" -P "${SCRIPT_DIRECTORY}/hbase_docker" \ + "${JDK_URL}"; then + echo "Error downloading Oracle JDK. Exiting..." >&2 + exit 1 + fi +fi + +# If MAVEN_URL is set, download Maven into the hbase_docker folder. +if [ -n "${MAVEN_URL}" ]; then + echo "Downloading Apache Maven..." + if ! wget -nv -N -P "${SCRIPT_DIRECTORY}/hbase_docker" "${MAVEN_URL}"; then + echo "Error downloading Apache Maven. Exiting..." >&2 + exit 1 + fi +fi + +# Before running docker build, confirm that the hbase_docker folder contains +# one JDK .tar.gz and one Maven .tar.gz. +FILE_CHECK_EXIT_CODE=0 +for file in jdk maven; do + NUM_FILES=$(ls -l "${SCRIPT_DIRECTORY}/hbase_docker/"*${file}*.tar.gz 2>/dev/null | \ + wc -l) + if [ ${NUM_FILES} -eq 0 ]; then + echo "Could not detect tarball containing \"${file}\" in hbase_docker folder." >&2 + FILE_CHECK_EXIT_CODE=1 + elif [ ${NUM_FILES} -gt 1 ]; then + echo "There are too many files containing \"${file}\" in hbase_docker folder." >&2 + FILE_CHECK_EXIT_CODE=1 + fi +done +if [ ${FILE_CHECK_EXIT_CODE} -ne 0 ]; then + echo "Required dependencies not satisfied. Exiting..." >&2 + exit 1 +fi + +# docker build can be buggy (e.g. see https://github.com/docker/docker/issues/4036). +# To get around this, this script will try doing up to ${MAX_BUILD_ATTEMPTS} builds. +BUILD_ATTEMPTS=0 +MAX_BUILD_ATTEMPTS=10 +echo "Beginning docker build..." +until docker build -t ${IMAGE_NAME} ${SCRIPT_DIRECTORY}/hbase_docker; do + ((++BUILD_ATTEMPTS)) + if [ ${BUILD_ATTEMPTS} -ge ${MAX_BUILD_ATTEMPTS} ]; then + echo "Build of ${IMAGE_NAME} failed after ${BUILD_ATTEMPTS} attempts. Exiting..." >&2 + exit 1 + else + echo "Build attempt #${BUILD_ATTEMPTS} of ${IMAGE_NAME} was unsuccessful. Retrying..." + fi +done + +echo "Successfully built ${IMAGE_NAME}." + +echo "Starting hbase shell..." +docker run -it ${IMAGE_NAME} diff --git a/dev-support/hbase_docker/Dockerfile b/dev-support/hbase_docker/Dockerfile new file mode 100644 index 0000000..5f87bfe --- /dev/null +++ b/dev-support/hbase_docker/Dockerfile @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:14.04 + +# Install Git, which is missing from the Ubuntu base images. +RUN apt-get update && apt-get install -y git + +# Add the dependencies from the hbase_docker folder and delete ones we don't need. +WORKDIR /root +ADD . /root +RUN find . -not -name "*tar.gz" -delete + +# Install Java. +RUN mkdir -p /usr/java +RUN tar xzf *jdk* --strip-components 1 -C /usr/java +ENV JAVA_HOME /usr/java + +# Install Maven. +RUN mkdir -p /usr/local/apache-maven +RUN tar xzf *maven* --strip-components 1 -C /usr/local/apache-maven +ENV MAVEN_HOME /usr/local/apache-maven + +# Add Java and Maven to the path. +ENV PATH /usr/java/bin:/usr/local/apache-maven/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +# Pull down HBase from Github and build it into /root/hbase-bin. +RUN git clone https://github.com/apache/hbase.git -b master +RUN mvn clean install -DskipTests assembly:single -f ./hbase/pom.xml +RUN mkdir -p hbase-bin +RUN tar xzf /root/hbase/hbase-assembly/target/*tar.gz --strip-components 1 -C /root/hbase-bin + +# Set HBASE_HOME, add it to the path, and start HBase. +ENV HBASE_HOME /root/hbase-bin +ENV PATH /root/hbase-bin/bin:/usr/java/bin:/usr/local/apache-maven/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +CMD ["/bin/bash", "-c", "start-hbase.sh; hbase shell"] diff --git a/dev-support/hbase_docker/README.md b/dev-support/hbase_docker/README.md new file mode 100644 index 0000000..0c09456 --- /dev/null +++ b/dev-support/hbase_docker/README.md @@ -0,0 +1,39 @@ +# hbase_docker + +## Overview + +The Dockerfile in this folder can be used to build a Docker image running +the latest HBase master branch in standalone mode. It does this by setting +up necessary dependencies, checking out the master branch of HBase from +GitHub, and then building HBase. By default, this image will start the HMaster +and launch the HBase shell when run. + +## Usage + +1. Download x64 .tar.gz files of the Oracle JDK and Apache Maven and place them + in this folder (i.e. both tarballs must be in the same folder as the + Dockerfile). Also note that the Dockerfile will properly pick up the tarballs + as long as the JDK file has "jdk" in its name and the Maven file contains + "maven". As an example, while developing this Dockerfile, my working directory + looked like this: + + ``` + $ ls -lh + total 145848 + -rw-r--r-- 1 root root 6956162 Sep 3 15:48 apache-maven-3.2.3-bin.tar.gz + -rw-r--r-- 1 root root 2072 Sep 3 15:48 Dockerfile + -rw-r--r-- 1 root root 142376665 Sep 3 15:48 jdk-7u67-linux-x64.tar.gz + -rw-r--r-- 1 root root 1844 Sep 3 15:56 README.md + ``` +2. Ensure that you have a recent version of Docker installed from + [docker.io](http://www.docker.io). +3. Set this folder as your working directory. +4. Type `docker build -t hbase_docker .` to build a Docker image called **hbase_docker**. + This may take 10 minutes or more the first time you run the command since it will + create a Maven repository inside the image as well as checkout the master branch + of HBase. +5. When this completes successfully, you can run `docker run -it hbase_docker` + to access an HBase shell running inside of a container created from the + **hbase_docker** image. Alternatively, you can type `docker run -it hbase_docker + bash` to start a container without a running HMaster. Within this environment, + HBase is built in /root/hbase-bin. -- 1.9.1