diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/pom.xml
index fa9a99b4cab..fd3cf3718a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/pom.xml
@@ -170,6 +170,73 @@
test
+
+
+
+ python-builder
+
+
+ PYTHON_MAWO_APP_LAUNCHER
+
+
+
+ localhost:50070
+ localhost:8088
+
+
+ whoami
+ False
+
+
+
+
+ org.codehaus.mojo
+ exec-maven-plugin
+ 1.6.0
+
+
+
+ python3
+ src/main/python3/helper-scripts
+
+ MaWo_Unittest_Runner.py
+ --namenode-http-add
+ ${namenode-http-add}
+ --rm-http-add
+ ${rm-http-add}
+ --docker-image
+ ${docker-image}
+ --registryurl
+ ${registryurl}
+ --mawo-app-user
+ ${mawo-app-user}
+ --skip-docker-image-creation
+ ${skip-docker-image-creation}
+
+
+ python_build
+ integration-test
+
+ exec
+
+
+
+
+
+ org.apache.rat
+ apache-rat-plugin
+ 0.12
+
+
+ src/main/python3/helper-scripts/mawo_hadoop_ut_launch.json
+
+
+
+
+
+
+
+
@@ -198,6 +265,16 @@
+
+ org.apache.rat
+ apache-rat-plugin
+ 0.12
+
+
+ src/main/python3/helper-scripts/mawo_hadoop_ut_launch.json
+
+
+
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/bin/mawo b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/bin/mawo
new file mode 100644
index 00000000000..7de6e5682e0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/bin/mawo
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`which $0`
+bindir=`dirname ${bin}`
+
+function usage(){
+ echo "Usage: mawo [--config conf] [COMMAND]"
+ echo " where COMMAND is one of:"
+ echo " master run the master daemon"
+ echo " worker run the worker daemon"
+ echo ""
+}
+
+if [ $# = 0 ]; then
+ usage
+ exit
+fi
+
+if [ $# -gt 1 ]
+then
+ if [ "--config" = "$1" ]
+ then
+ shift
+ confdir=$1
+ if [ ! -d "$confdir" ]; then
+ echo "Error: Cannot find configuration directory: $confdir"
+ exit 1
+ fi
+ shift
+ CONF_DIR=$confdir
+ fi
+fi
+
+COMMAND=$1
+case $COMMAND in
+ --help|-help|-h)
+ usage
+ exit
+ ;;
+
+ master|worker)
+ if [ "$COMMAND" = "master" ] ; then
+ CLASS=org.apache.hadoop.applications.mawo.server.master.Master
+ shift
+ elif [ "$COMMAND" = "worker" ] ; then
+ CLASS=org.apache.hadoop.applications.mawo.server.worker.Worker
+ shift
+ fi
+ ;;
+esac
+
+JAVA=java
+JAVA_OPTS="-Xmx3072m -Xms1024m"
+export MAWO_HOME=${MAWO_HOME:-$bindir/../}
+LIBS_DIR=$MAWO_HOME/lib/*
+
+export CLASSPATH=$CONF_DIR:$MAWO_HOME/*:$LIBS_DIR
+echo $CLASSPATH
+
+echo "$JAVA" $JAVA_OPTS $CLASS "$@"
+exec "$JAVA" $JAVA_OPTS $CLASS "$@"
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/DockerImage/Dockerfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/DockerImage/Dockerfile
new file mode 100644
index 00000000000..35a74e60d51
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/DockerImage/Dockerfile
@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for running Hadoop UT using MaWo application.
+
+
+FROM docker.io/centos:6.8
+
+#################################################################################
+# Install basic stuff
+#################################################################################
+RUN yum install -y epel-release && \
+ yum install -y wget && \
+ yum install -y gcc && \
+ yum install -y gcc-c++
+
+RUN yum install -y java-1.8.0-openjdk-devel && \
+ yum install -y iproute && \
+ yum install -y telnet bind-tools net-tools && \
+ yum install -y pdsh && \
+ yum install -y rsync
+
+#################################################################################
+# nobody user's home
+#################################################################################
+RUN mkdir /home/nobody && \
+ chown -R nobody:nobody /home/nobody && \
+ chsh -s /bin/bash nobody && \
+ usermod -d /home/nobody nobody
+
+#################################################################################
+# Install stuff needed for compilation
+#################################################################################
+# maven and git
+RUN wget http://repos.fedorapeople.org/repos/dchen/apache-maven/epel-apache-maven.repo \
+ -O /etc/yum.repos.d/epel-apache-maven.repo
+RUN yum install -y apache-maven git
+
+#################################################################################
+# Install protobuf needed for Hadoop compilation
+#################################################################################
+RUN yum install -y gcc-c++
+RUN cd /tmp && \
+ wget https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.bz2 && \
+ tar xfvj protobuf-2.5.0.tar.bz2 && \
+ cd protobuf-2.5.0 && \
+ ./configure && make && make install
+
+################################################################################
+# Setup for MaWo
+################################################################################
+
+RUN mkdir /tmp/mawo && \
+ mkdir /tmp/mawo/logs
+
+COPY hadoop-applications-mawo-*.tar.gz /tmp/mawo
+
+RUN cd /tmp/mawo && \
+ tar -xvf hadoop-applications-mawo-*.tar.gz && \
+ chmod -R 777 /tmp/mawo
+
+#################################################################################
+# Setup Apache Hadoop Source code
+#################################################################################
+RUN mkdir /hadoop-src && \
+ cd /hadoop-src
+
+COPY hadoop-src.tar.gz /hadoop-src/
+
+RUN cd /hadoop-src && \
+ ls -la && \
+ tar -xvf hadoop-src.tar.gz
+
+RUN cd /hadoop-src/hadoop && mvn clean install -DskipTests -DskipITs
+
+ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
+
+WORKDIR /tmp/mawo/hadoop-applications-mawo-0.0.1-SNAPSHOT/bin
+
+ENTRYPOINT ["./mawo"]
+
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/GeneratePayloadFile.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/GeneratePayloadFile.py
new file mode 100644
index 00000000000..b04e024ce6a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/GeneratePayloadFile.py
@@ -0,0 +1,241 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import defaultdict
+import os
+import fnmatch
+import re
+import sys
+import json
+from Task import Task
+
+class GeneratePayloadFile:
+ _project = None
+ _ut_config = None
+ _base_source_dir = None
+ _exclude_tests = None
+ _include_tests = None
+ _java_pattern = None
+
+ def __init__(self, project, ut_config, base_source_dir, payload_file, exclude_tests="", include_tests="", java_pattern="Test*.java"):
+ """
+ Initialize Generate Payload File Class
+ :param project: project name which is 'hadoop'
+ :param ut_config: unit test configuration file (hadoop_ut.ini)
+ :param base_source_dir: The path to hadoop source code
+ :param payload_file: Destination payload file
+ :param exclude_tests: A comma seperated list of tests which need to excluded from execution (TODO: not yet supported)
+ :param include_tests: A comma seperated list of tests which need to included from execution (TODO: not yet supported)
+ :param java_pattern: a pattern to find hadoop test files
+ """
+ self._project = project
+ self._base_source_dir = base_source_dir
+ self._ut_config = ut_config
+ self._exclude_tests = exclude_tests
+ self._include_tests = include_tests
+ self._java_pattern = java_pattern
+ self._payload_file = payload_file
+
+ def get_tests(self):
+ '''
+ Find out tests from Hadoop Source code
+ :return:
+ loaded_tests : A dict which has a dictionary with module and test mapping
+ {"module1":[test1, test2], "module2":[test4, test5]}
+ tests_list : A list of test cases. [test1, test2, test3, test4]
+ '''
+ loaded_tests = defaultdict(list)
+ tests_list = []
+
+ exclude_tests_list = self._exclude_tests.split(",")
+ include_tests_list = self._include_tests.split(",")
+
+ pattern_list = []
+ pattern_list.extend("".join(self._java_pattern.split()).split(("|")))
+
+
+ for dirname, dirnames, filenames in os.walk(self._base_source_dir):
+ for extention in pattern_list:
+ for filename in fnmatch.filter(filenames, extention):
+ filename = re.sub('\.%s$' % extention.split(".")[-1], '', filename)
+ tests_list.append(filename)
+
+ path_components = dirname.split('/')
+ module = 'SPECIAL_BUCKET'
+ for idx, val in enumerate(path_components):
+ if path_components[idx] == 'src' and (idx + 1) < len(path_components) and \
+ path_components[idx + 1] == 'test' and (idx - 1 >= 0):
+ module = path_components[idx - 1]
+ break
+ if self._project not in module:
+ if len(include_tests_list) > 1:
+ if filename in include_tests_list:
+ loaded_tests["SPECIAL_BUCKET"].append(filename)
+ else:
+ if filename not in exclude_tests_list:
+ loaded_tests["SPECIAL_BUCKET"].append(filename)
+ else:
+ if len(include_tests_list) > 1:
+ if filename in include_tests_list:
+ loaded_tests[module].append(filename)
+ else:
+ if filename not in exclude_tests_list:
+ loaded_tests[module].append(filename)
+
+ return loaded_tests, tests_list
+
+ def create_buckets(self, loaded_tests):
+ """
+ Create N buckets for Hadoop UT execution
+ :param loaded_tests: A dict which has a dictionary with module and test mapping
+ {"module1":[test1, test2], "module2":[test4, test5]}
+ :return: buckets = ['module1', 'module2', 'SPECIAL_BUCKET=testa,testb']
+ """
+ buckets = [modules for modules in loaded_tests.keys()]
+ if "SPECIAL_BUCKET" in buckets:
+ special_tests = loaded_tests["SPECIAL_BUCKET"]
+ print(special_tests)
+ buckets.remove("SPECIAL_BUCKET")
+ buckets.append("SPECIAL_BUCKET=" + ",".join(special_tests))
+ return buckets
+
+ def get_task_env(self):
+ """
+ Get TaskEnv values to execute Hadoop UT
+ :return:
+ """
+ env_dict = {}
+ try:
+ env_str = self._ut_config.get(self._project, 'all_tasks_env')
+ envs = env_str.split(",")
+ for env in envs:
+ key, value = env.split("=", 1)
+ try:
+ if value.startswith('$'):
+ env_dict[key] = os.environ[value[1:]]
+ else:
+ env_dict[key] = value
+ except:
+ env_dict[key] = "None"
+
+ return env_dict
+ except:
+ print("Get environment failed with Exception : %s" % sys.exc_info()[0])
+ return env_dict
+
+ def update_worker_source_code_dir(self, str):
+ """
+ Replace $(worker_source_code_dir) with correct value
+ :param str:
+ :return:
+ """
+ return str.replace("$(worker_source_code_dir)", self._ut_config.get(self._project, "worker_source_code_dir"))
+
+ def get_task_timeout(self):
+ """
+ Get task timeout
+ :return:
+ """
+ return int(self._ut_config.get(self._project, "task_timeout"))
+
+
+ def create_setup_task(self):
+ """
+ Create setup Task
+ :return:
+ """
+ setup_cmd = self._ut_config.get(self._project, "setup_task")
+ setup_cmd = self.update_worker_source_code_dir(setup_cmd)
+ setup_task = Task("SetupTaskId", task_cmd=setup_cmd,
+ execution_time=self.get_task_timeout(),
+ env=self.get_task_env())
+ return setup_task
+
+ def create_teardown_task(self):
+ """
+ Create teardown Task
+ :return:
+ """
+ teardown_cmd = self._ut_config.get(self._project, "teardown_task")
+ teardown_cmd = self.update_worker_source_code_dir(teardown_cmd)
+ teardown_task = Task("TeardownTaskId", task_cmd=teardown_cmd,
+ execution_time=self.get_task_timeout(), env=self.get_task_env())
+ return teardown_task
+
+ def create_tasks(self, buckets):
+ """
+ Generate Tasks as per hadoop project
+ :param buckets:
+ :return:
+ """
+ task_list = []
+ base_task_cmd = self._ut_config.get(self._project, "base_task_cmd")
+ for bucket in buckets:
+ if bucket.find("SPECIAL_BUCKET") >= 0:
+ tests = bucket.split("=")[1]
+ task_cmd = base_task_cmd.replace("--projects :", "-Dtest=" + tests)
+ else:
+ task_cmd = base_task_cmd.replace("", bucket)
+ task_cmd = self.update_worker_source_code_dir(task_cmd)
+ task_list.append(Task(bucket, task_cmd=task_cmd,
+ execution_time=self.get_task_timeout(), env=self.get_task_env()))
+ return task_list
+
+ def build_payload_file(self, setup_task, teardown_task, tasks):
+ """
+ Build Payload file for MaWo Application
+ :param setup_task:
+ :param teardown_task:
+ :param tasks:
+ :return:
+ """
+ if (os.path.exists(self._payload_file)):
+ os.remove(self._payload_file)
+
+ output_json = defaultdict(list)
+
+ output_json['SetupTask'] = {'TaskCmd': setup_task.get_task_cmd(),
+ 'TaskEnv': setup_task.get_env(),
+ 'TaskTimeout': setup_task.get_execution_time()}
+
+ output_json['TeardownTask'] = { 'TaskCmd': teardown_task.get_task_cmd(),
+ 'TaskEnv': teardown_task.get_env(),
+ 'TaskTimeout': teardown_task.get_execution_time()}
+
+ for task in tasks:
+ output_json['Tasks'].append({
+ 'TaskCmd': task.get_task_cmd(),
+ 'TaskEnv': task.get_env(),
+ 'TaskTimeout': task.get_execution_time()
+ })
+
+ with open(self._payload_file, 'w') as outfile:
+ json.dump(output_json, outfile, sort_keys=True, indent=4,
+ separators=(',', ': '))
+
+ def generate_payload_file(self):
+ """
+ Generate payload file for Hadoop UT.
+ :return: payload file, expected number of workers
+ """
+ loaded_tests, tests_list = self.get_tests()
+ buckets = self.create_buckets(loaded_tests)
+ self.build_payload_file(self.create_setup_task(), self.create_teardown_task(), self.create_tasks(buckets))
+ print("***************************** Worker Recommendation ******************************")
+ print("MaWo Payload file is present at %s.\n" % self._payload_file)
+ print("This input file has %d buckets. \nRecommendation: Launch MaWo app with %d workers" % (len(buckets), len(buckets)))
+ print("***********************************************************************************")
+ return self._payload_file, len(buckets)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/MaWoLauncher.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/MaWoLauncher.py
new file mode 100644
index 00000000000..472bac21f64
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/MaWoLauncher.py
@@ -0,0 +1,167 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import json
+import subprocess
+import random
+
+class MaWoLauncher:
+ hdfs_url = None
+ rm_url = None
+ mawo_app_user = None
+ _headers = "Content-Type: application/json"
+ cmd_executor = None
+
+ def __init__(self, hdfs_url, rm_url, mawo_app_user, cmd_executor):
+ """
+ MaWoLauncher class initializer
+ :param hdfs_url: dfs.namenode.http-address property from Hadoop cluster
+ :param rm_url: yarn.resourcemanager.webapp.address property from Hadoop cluster
+ :param mawo_app_user: User which need to launch MaWo app in Hadoop cluster
+ :param cmd_executor: instance of cmd executor class to run cmds
+ """
+ self.hdfs_url = hdfs_url
+ self.rm_url = rm_url
+ self.mawo_app_user = mawo_app_user
+ self.cmd_executor = cmd_executor
+
+ def get_mawo_app_url(self, service_name=None):
+ """
+ Get RM URL to access yarn service
+ :param service_name: If service name is passed, RM url for that service is returned
+ otherwise generic RM url is returned
+ :return:
+ """
+ if not service_name:
+ return "http://%s/app/v1/services?user.name=%s" % (self.rm_url, self.mawo_app_user)
+ else:
+ return "http://%s/app/v1/services/%s?user.name=%s" % (self.rm_url, service_name, self.mawo_app_user)
+
+ def launch_mawo_app(self, mawo_launch_json):
+ """
+ Launch MaWo application in Hadoop Cluster
+ :param mawo_launch_json: launch file for mawo app
+ :return:
+ """
+ cmd = 'curl --silent -k -X POST -H "%s" --negotiate -u : -d @%s %s'\
+ % (self._headers, mawo_launch_json, self.get_mawo_app_url())
+ returnCode, stdout = self.cmd_executor.execute_cmd(cmd)
+ assert returnCode == 0, "submit MaWo application to Hadoop cluster"
+
+ def upload_payload_file_to_hdfs(self, payload_file_path):
+ """
+ Upload mawo payload file to HDFS
+ :param payload_file_path: The local path to payload file.
+ This file will be added to /user/ in HDFS
+ :return:
+ """
+ payload_file = os.path.basename(payload_file_path)
+ cmd_1 = 'curl -i -X PUT "http://%s/webhdfs/v1/user/%s/%s?user=%s&op=CREATE"' \
+ % (self.hdfs_url, self.mawo_app_user, payload_file, self.mawo_app_user)
+ returnCode, stdout = self.cmd_executor.execute_cmd(cmd_1)
+ assert returnCode == 0, "curl put cmd failed"
+
+ url = None
+ str_stdout = stdout.decode('ASCII')
+ for line in str_stdout.split("\r\n"):
+ if line.find("Location") >= 0:
+ url = line.split("Location:")[1]
+ break
+
+ if url:
+ url = url + "&user.name=%s" % self.mawo_app_user
+ url = url.strip()
+ else:
+ print("Upload to HDFS Failed")
+ sys.exit(2)
+
+ cmd_2 = 'curl -i -T %s "%s"' % (payload_file_path, url)
+ returnCode, stdout = self.cmd_executor.execute_cmd(cmd_2)
+ assert returnCode == 0, "curl -T command failed"
+
+ def get_mawo_app_details(self, service_name=None):
+ """
+ Get yarn service status with rest api
+ :param service_name: yarn service name
+ :return: yarn service status json object
+ """
+ try:
+ local_app_details = service_name + "-" + str(random.randint(1, 1000)) + ".json"
+ cmd = 'curl -X GET %s > %s' % (self.get_mawo_app_url(service_name), local_app_details)
+ returnCode, stdout = self.cmd_executor.execute_cmd(cmd)
+ assert returnCode == 0, "Get MaWp app details failed"
+ data = None
+ with open(local_app_details) as f:
+ data = json.load(f)
+ os.remove(local_app_details)
+ return data
+ except Exception as e:
+ print("get yarn service details failed.")
+ print(e)
+ return None
+
+
+class CmdExecutor:
+
+ def __init__(self):
+ pass
+
+ def get_root_user(self):
+ """
+ return root user
+ :return:
+ """
+ return "root"
+
+ def sudocmd(self, cmd, user, env=None, doEscapeQuote=True):
+ """
+ Create sudo cmd
+ :param cmd:
+ :param user:
+ :param env:
+ :param doEscapeQuote:
+ :return:
+ """
+ runCmd = ''
+ if env:
+ for key in env.keys():
+ runCmd += 'export %s=%s;' % (key, env[key])
+ if doEscapeQuote:
+ runCmd += cmd.replace('"', '\\"')
+ else:
+ runCmd += cmd
+ return "sudo su - -c \"%s\" %s" % (runCmd, user)
+
+ def execute_cmd(self, cmd, user=None, cwd=None):
+ """
+ Execute a command locally
+ :param cmd:
+ :param user:
+ :param cwd:
+ :return:
+ """
+ if cwd:
+ cmd = "cd %s && %s" % (cwd, cmd)
+ if user:
+ cmd = self.sudocmd(cmd, user)
+ print("RUNNING: %s" % cmd)
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, cwd=cwd)
+ output, _ = proc.communicate()
+ print("OUTPUT: \n")
+ print(output)
+ return proc.returncode, output
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/MaWo_Unittest_Runner.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/MaWo_Unittest_Runner.py
new file mode 100644
index 00000000000..48ba8a31e0d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/MaWo_Unittest_Runner.py
@@ -0,0 +1,250 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import getpass
+import os
+import configparser
+import random
+from shutil import copyfile
+from GeneratePayloadFile import GeneratePayloadFile
+from argparse import ArgumentParser
+from MaWoLauncher import MaWoLauncher, CmdExecutor
+
+parser = ArgumentParser()
+
+parser.add_argument(
+ '--namenode-http-add',
+ dest='hdfs_url',
+ action='store',
+ type=str,
+ required=True,
+ help='Enter dfs.namenode.http-address property for Hadoop cluster')
+
+parser.add_argument(
+ '--rm-http-add',
+ dest='rm_url',
+ action='store',
+ type=str,
+ required=True,
+ help='Enter yarn.resourcemanager.webapp.address property for Hadoop cluster')
+
+parser.add_argument(
+ '--mawo-app-user',
+ dest='mawo_app_user',
+ action='store',
+ type=str,
+ default="whoami",
+ help='Enter user which will be launching MaWo app on Hadoop cluster')
+
+parser.add_argument(
+ '--docker-image',
+ dest='mawo_docker_image',
+ action='store',
+ type=str,
+ required=True,
+ help='Enter docker imagename:tag for MaWo app')
+
+parser.add_argument(
+ '--registryurl',
+ dest='registryurl',
+ action='store',
+ type=str,
+ required=True,
+ help='Enter Registry url. "/" will be used for MaWo app configuration')
+
+parser.add_argument(
+ '--skip-docker-image-creation',
+ dest='skip_docker_image_creation',
+ action='store',
+ type=str,
+ default='False',
+ help='Use --skip-docker-image-creation True to skip docker image creation')
+
+
+class MaWoDriver:
+ mawo_docker_image = None
+ mawo_docker_image_tag = None
+ mawo_app_user = None
+ cmd_executor = None
+
+ def __init__(self, cmd_executor, mawo_app_user, mawo_docker_image, mawo_docker_image_tag="latest"):
+ """
+ MaWoDriver class initialization
+ :param cmd_executor: instance of cmd executor class to run cmds
+ :param mawo_app_user: User which need to launch MaWo app in Hadoop cluster
+ :param mawo_docker_image: Docker image name which need to be build to launch Hadoop UT
+ :param mawo_docker_image_tag: docker image version tag
+ """
+ self.mawo_docker_image = mawo_docker_image
+ self.mawo_docker_image_tag = mawo_docker_image_tag
+ self.mawo_app_user = mawo_app_user
+ self.cmd_executor = cmd_executor
+
+ def get_hadoop_src_rootpath(self):
+ """
+ return hadoop src code path "/home//hadoop"
+ :return:
+ """
+ return os.path.join("/home", getpass.getuser(), "hadoop")
+
+ def get_mawo_application_src_path(self):
+ """
+ return mawo application src path
+ :return:
+ """
+ return os.path.join(self.get_hadoop_src_rootpath(), "hadoop-yarn-project", "hadoop-yarn",
+ "hadoop-yarn-applications", "hadoop-yarn-applications-mawo")
+
+ def get_mawo_docker_image_dir(self):
+ """
+ return mawo application docker image dir
+ :return:
+ """
+ return os.path.join(self.get_mawo_application_src_path(), "src", "main", "python3", "helper-scripts",
+ "DockerImage")
+
+ def get_mawo_hadoop_ut_launch(self):
+ """
+ return json file to launch mawo hadoop UT
+ :return:
+ """
+ return os.path.join(self.get_mawo_application_src_path(), "src", "main", "python3", "helper-scripts",
+ "mawo_hadoop_ut_launch.json")
+
+ def create_hadoop_src_tarball(self):
+ """
+ Create Hadoop src tarball file and move under DockerImage dir
+ :return:
+ """
+ hadoop_src_path_local = "/home/%s" % getpass.getuser()
+ cmd = "tar -zcvf hadoop-src.tar.gz hadoop/"
+ returnCode, stdout = self.cmd_executor.execute_cmd(cmd, user=self.cmd_executor.get_root_user(),
+ cwd=hadoop_src_path_local)
+ assert returnCode == 0, "create of hadoop-src.tar.gz failed"
+ cmd = "mv hadoop-src.tar.gz %s" % (self.get_mawo_docker_image_dir())
+ returnCode, stdout = self.cmd_executor.execute_cmd(cmd, user=self.cmd_executor.get_root_user(),
+ cwd=hadoop_src_path_local)
+ assert returnCode == 0, "move of hadoop-src.tar.gz to DockerImage location failed"
+
+ def create_mawo_app_tar_ball(self):
+ """
+ Create mawo app tarball and move under DockerImage dir
+ :return:
+ """
+ cmd = "mvn clean install -DskipTests"
+ returnCode, stdout = self.cmd_executor.execute_cmd(cmd, user=self.cmd_executor.get_root_user(),
+ cwd=self.get_mawo_application_src_path())
+ assert returnCode == 0, "mvn install for mawo app failed"
+ cmd = "mv %s/target/hadoop-applications-mawo-*.tar.gz %s" \
+ % (self.get_mawo_application_src_path(), self.get_mawo_docker_image_dir())
+ returnCode, stdout = self.cmd_executor.execute_cmd(cmd, user=self.cmd_executor.get_root_user(),
+ cwd=self.get_mawo_application_src_path())
+ assert returnCode == 0, "mv of hadoop-application-mawo-*.tar.gz failed"
+
+ def build_mawo_docker_image(self):
+ """
+ Build mawo docker image
+ :return:
+ """
+ cmd = "docker build -t %s:%s ." % (self.mawo_docker_image, self.mawo_docker_image_tag)
+ returnCode, stdout = self.cmd_executor.execute_cmd(cmd, user=self.cmd_executor.get_root_user(),
+ cwd=self.get_mawo_docker_image_dir())
+ assert returnCode == 0, "docker build cmd failed"
+
+ def create_docker_image(self):
+ """
+ An orchestrator function to create docker image
+ :return:
+ """
+ # create tar ball for hadoop-src
+ print("****** Create mawo app tar ball ******")
+ self.create_mawo_app_tar_ball()
+
+ print("****** Create Hadoop src code tarball *******")
+ # create hadoop-src tarball
+ self.create_hadoop_src_tarball()
+
+ print("****** Build Docker image *******")
+ print("##### Beware this step may take long time because hadoop compile is involved in this step #####")
+ # build docker image
+ self.build_mawo_docker_image()
+
+ def update_mawo_configuration(self, registry_name, docker_image, rm_host, num_of_workers):
+ """
+ Update MaWo Configuration to put proper RMHOST and DOCKERIMAGE
+ :return:
+ """
+ dst_file = os.path.join(os.path.dirname(self.get_mawo_hadoop_ut_launch()),
+ "mawo-launch-" + str(random.randint(1, 1000)) + ".json")
+ copyfile(self.get_mawo_hadoop_ut_launch(), dst_file)
+ cmd = "sed -i -e 's//%s/g' %s" % (rm_host, dst_file)
+ returnCode, stdout = self.cmd_executor.execute_cmd(cmd)
+ assert returnCode == 0, "update RMHost to %s failed" % dst_file
+ registry_name = registry_name + "/" + docker_image
+ escaped_registry_image = registry_name.replace("/", "\/")
+ cmd = "sed -i -e 's//%s/g' %s" % (escaped_registry_image, dst_file)
+ returnCode, stdout = self.cmd_executor.execute_cmd(cmd)
+ assert returnCode == 0, "update RMHost to %s failed" % dst_file
+ cmd = "sed -i -e 's//%s/g' %s" % (num_of_workers, dst_file)
+ returnCode, stdout = self.cmd_executor.execute_cmd(cmd)
+ assert returnCode == 0, "update WORKERCONTAINERS to %s failed" % dst_file
+ return dst_file
+
+
+
+def main():
+ args = parser.parse_args()
+ print("******** Execution of MaWo Unittest Runner Starts **********")
+ cmd_executor = CmdExecutor()
+
+ if args.mawo_app_user == "whoami":
+ args.mawo_app_user = getpass.getuser()
+
+ if len(args.mawo_docker_image.split(":")) == 2:
+ docker_image, docker_tag = args.mawo_docker_image.split(":")
+ md = MaWoDriver(cmd_executor, args.mawo_app_user, docker_image, docker_tag)
+ else:
+ md = MaWoDriver(cmd_executor, args.mawo_app_user, args.mawo_docker_image)
+
+ if args.skip_docker_image_creation.lower() != "true":
+ md.create_docker_image()
+
+ print("Upload docker image to registry at %s/%s" % (args.registryurl, args.mawo_docker_image))
+ input("Press Enter after its done to proceed")
+
+ print("**** Create MaWo Payload file ****")
+ unit_test_config = configparser.ConfigParser()
+ unit_test_config.read(os.path.abspath("hadoop_ut.ini"))
+ gp = GeneratePayloadFile("hadoop", unit_test_config, md.get_hadoop_src_rootpath(), "mawo-payload.json")
+ payload_file_path, num_of_workers = gp.generate_payload_file()
+
+ mawo_launch_json = md.update_mawo_configuration(args.registryurl, args.mawo_docker_image,
+ args.rm_url.split(':')[0], num_of_workers)
+ print("MaWo launch file = %s is created" % mawo_launch_json)
+
+ ml = MaWoLauncher(args.hdfs_url, args.rm_url, args.mawo_app_user, cmd_executor)
+
+ print("**** Upload mawo payload file to HDFS ****")
+ ml.upload_payload_file_to_hdfs(payload_file_path)
+
+ print("*** Launch MaWo App ***")
+ ml.launch_mawo_app(mawo_launch_json)
+
+ output = ml.get_mawo_app_details('mawo-hadoop-ut')
+ print(output['name'])
+
+if __name__ == "__main__":
+ main()
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/Task.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/Task.py
new file mode 100644
index 00000000000..02af9c6229d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/Task.py
@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class Task:
+ _task_id = None
+ _task_cmd = None
+ _task_group_id = None
+ _execution_time = 0
+ _env = {}
+
+ def __init__(self, task_id, task_cmd, execution_time=0, group_name=None, env={}):
+ self._task_id = task_id
+ self._task_cmd = task_cmd
+ self._execution_time = execution_time
+ self._task_group_id = group_name
+ self._env = env
+
+ def get_task_id(self):
+ '''
+ get task name
+ :return:
+ '''
+ return self._task_id
+
+ def set_task_id(self, task_id):
+ '''
+ set task id
+ :return:
+ '''
+ self._task_id = task_id
+
+ def get_task_cmd(self):
+ '''
+ get task cmd
+ :return:
+ '''
+ return self._task_cmd
+
+ def set_task_cmd(self, task_cmd):
+ '''
+ Set task cmd
+ :param task_cmd:
+ :return:
+ '''
+ self._task_cmd = task_cmd
+
+ def get_task_group_id(self):
+ '''
+ get group name for Task
+ :return:
+ '''
+ if self._task_group_id:
+ return self._task_group_id
+ else:
+ return None
+
+ def set_task_group_id(self, group_name):
+ '''
+ set group for task
+ :return:
+ '''
+ self._task_group_id = group_name
+
+ def get_execution_time(self):
+ '''
+ Get execution-time for a task
+ :return:
+ '''
+ return self._execution_time
+
+ def set_execution_time(self, execution_time):
+ '''
+ Get execution-time for a task
+ :return:
+ '''
+ self._execution_time = execution_time
+
+ def set_env(self, env):
+ '''
+ set env variable
+ :param env:
+ :return:
+ '''
+ self._env = env
+
+ def get_env(self):
+ '''
+ Get env value
+ :return:
+ '''
+ return self._env
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/hadoop_ut.ini b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/hadoop_ut.ini
new file mode 100644
index 00000000000..7bba5e2d77a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/hadoop_ut.ini
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#################################################################################
+[hadoop]
+#################################################################################
+
+worker_source_code_dir = /hadoop-src/hadoop
+
+# The actual unit-test commands
+base_task_cmd = cd $(worker_source_code_dir); mvn -B -nsu test -Dmaven.test.failure.ignore=true --projects :
+setup_task = bash -c "cd $(worker_source_code_dir) && mvn install -DskipTests -DskipIT"
+teardown_task =
+all_tasks_env = JAVA_OPTS=-Xms2g -Xmx4g -XX:MaxPermSize=1024m -verbose:gc -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit,MAVEN_OPTS=-Xmx4096m,JDK_VERSION=1.8.0_181
+task_timeout = 14400
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/mawo_hadoop_ut_launch.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/mawo_hadoop_ut_launch.json
new file mode 100644
index 00000000000..f4824692799
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts/mawo_hadoop_ut_launch.json
@@ -0,0 +1,72 @@
+{
+ "name": "mawo-hadoop-ut",
+ "artifact": {
+ "type": "DOCKER",
+ "id": ""
+ },
+ "configuration": {
+ "env": {
+ "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK": "hadoop",
+ "YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE": "true"
+ },
+ "properties": {
+ "docker.network": "hadoop"
+ }
+ },
+ "components": [{
+ "dependencies": [],
+ "resource": {
+ "memory": "2048",
+ "cpus": "1"
+ },
+ "name": "master",
+ "run_privileged_container": true,
+ "number_of_containers": 1,
+ "launch_command": "--config,/tmp/mawo-config,master,-job_descripiton_file_path,/tmp/mawo/mawo-payload.json,-mawo_app_name,mawo-hadoop-ut",
+ "configuration": {
+ "env":{
+ "master_tasks_status_log_path": ""
+ },
+ "files": [{
+ "type": "PROPERTIES",
+ "dest_file": "/tmp/mawo-config/mawo.properties",
+ "properties": {
+ "mawo.job-builder.class": "org.apache.hadoop.applications.mawo.server.master.job.SimpleTaskJsonJobBuilder",
+ "ycloud.url": "http://:9191",
+ "rpc.server.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}"
+ }
+ },
+ {
+ "type":"TEMPLATE",
+ "dest_file": "/tmp/mawo/mawo-payload.json",
+ "src_file": "mawo-payload.json"
+ }]
+ }
+ }, {
+ "dependencies": ["master"],
+ "resource": {
+ "memory": "8072",
+ "cpus": "1"
+ },
+ "name": "worker",
+ "run_privileged_container": true,
+ "number_of_containers": ,
+ "launch_command": "--config,/tmp/mawo-config,worker",
+ "configuration": {
+ "env":{
+ "worker_workspace": ""
+ },
+ "files": [{
+ "type": "PROPERTIES",
+ "dest_file": "/tmp/mawo-config/mawo.properties",
+ "properties": {
+ "worker.num.tasks": "1",
+ "ycloud.url": "http://:9191",
+ "rpc.server.hostname": "master-0.${SERVICE_NAME}.${USER}.${DOMAIN}"
+ }
+ }]
+ }
+ }],
+ "lifetime": -1,
+ "version": 1.0
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/MasterWorkerApp.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/MasterWorkerApp.md
new file mode 100644
index 00000000000..f9974c615c8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/MasterWorkerApp.md
@@ -0,0 +1,80 @@
+
+
+
+
+MaWo: A Master Worker framework on YARN Services
+===================================================
+
+Overview
+---------
+There are existing frameworks on YARN which can be used to run a job in distributed manner such as Mapreduce, Tez, Spark etc. But master-worker use-cases usually are force-fed into one of these existing frameworks which have been designed primarily around data-parallelism instead of generic Master Worker type of computations. MaWo is a YARN service based framework which handles Master Worker based workload. The overall goal is to create an app that can take an input job specification with tasks, their expected durations and have a Master dish the tasks off to a predetermined set of workers. The components will be responsible to finish the job within specific time duration.
+
+MaWo Components
+------------------
+MaWo app is a YARN Service Application. It has mainly two components
+
+# Master
+ - Read MaWo-Payload file and create a queue of Tasks
+ - Register Worker
+ - Assign tasks to worker nodes
+ - Monitor status of Tasks
+ - Log Task status
+
+# Worker
+ - Send heartbeat to Worker
+ - Execute Task
+
+# MaWo-Payload File
+This is a json file which consist of Setup Task , a list of Tasks and a Teardown Task. Setup task gets executed first on all workers. A list of Tasks are dished off to N workers. Teardown task gets executed after all tasks execution is finished.
+
+Here, each Task consist of three entities.
+- TaskCmd : Cmd to be executed
+- TaskEnv : Environment variables needed to execute a task
+- TaskTimeout: If task execution exceeds task timeout (its in seconds), a task is aborted and marked as Failure.
+
+How to Run Hadoop UT using MaWo
+---------------------------------
+Pre-requisite:
+- A Hadoop 3.x cluster
+- Enable Docker on YARN
+
+Follow below steps from your local machine / laptop:
+
+Required Fields:
+- namenode-http-add = Put dfs.namenode.http-address property value from Hadoop 3.x cluster
+- rm-http-add = Put yarn.resourcemanager.webapp.addres property value from Hadoop 3.x cluster
+- docker-image = A docker image need to be created to launch MaWo app. Pass docker-image-name:version
+- registryurl = registry url on which above docker image will be hosted
+
+Optinal Fields:
+- skip-docker-image-creation = If you already have a Mawo docker image ready, you can skip this step by using -Dskip-docker-image-creation=True . Its default value is False
+- mawo-app-user = Pass user which need to launch MaWo app in Hadoop cluster. Its default value is "whoami" which means that Mawo app will be launched as user which started start-build-env.sh
+
+
+Using Maven
+-------------
+```
+sh start-build-env.sh
+cd hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo
+mvn integration-test -DPYTHON_MAWO_APP_LAUNCHER -Dnamenode-http-add= -Drm-http-add= -Dmawo-app-user=whoami -Ddocker-image=ut-mawo-test:latest -Dregistryurl= -Dskip-docker-image-creation=False
+```
+
+Using Python3
+---------------
+```
+sh start-build-env.sh
+cd hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/src/main/python3/helper-scripts
+python3 MaWo_Unittest_Runner.py --namenode-http-add --rm-http-add --mawo-app-user whoami --docker-image ut-mawo-test:latest --registryurl
+```
diff --git a/start-build-env.sh b/start-build-env.sh
index bf6b4113cd2..69a6c106652 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -61,7 +61,12 @@ docker build -t "hadoop-build-${USER_ID}" - < "/etc/sudoers.d/hadoop-build-${USER_ID}"
+RUN echo "${USER_NAME}\tALL=NOPASSWD: ALL" > "/etc/sudoers.d/hadoop-build-${USER_ID}"
+RUN apt-get -y remove docker docker-engine docker.io && \
+ apt-get update && apt-get install -y apt-transport-https ca-certificates wget software-properties-common && \
+ wget https://download.docker.com/linux/ubuntu/gpg && apt-key add gpg && \
+ echo 'deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable' | sudo tee -a /etc/apt/sources.list.d/docker.list && \
+ apt-get update && apt-get -y install docker-ce
ENV HOME /home/${USER_NAME}
UserSpecificDocker
@@ -78,5 +83,6 @@ docker run --rm=true $DOCKER_INTERACTIVE_RUN \
-v "${PWD}:/home/${USER_NAME}/hadoop${V_OPTS:-}" \
-w "/home/${USER_NAME}/hadoop" \
-v "${HOME}/.m2:/home/${USER_NAME}/.m2${V_OPTS:-}" \
+ -v /var/run/docker.sock:/var/run/docker.sock \
-u "${USER_NAME}" \
"hadoop-build-${USER_ID}" "$@"