commit 77875e38d789ccf9fd362920574dde265a43dc51 Author: Eric Yang Date: Wed Apr 3 14:51:04 2019 -0400 YARN-9254. Added support for storing application catalog data to HDFS. Contributed by Eric Yang diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-docker/src/main/scripts/entrypoint.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-docker/src/main/scripts/entrypoint.sh index 1666063..81dd838 100755 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-docker/src/main/scripts/entrypoint.sh +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-docker/src/main/scripts/entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -x # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -32,14 +32,36 @@ template_generator() { export JAVA_HOME=/usr/lib/jvm/jre export HADOOP_CONF_DIR=/etc/hadoop/conf -/opt/apache/solr/bin/solr start -p 8983 -force -/opt/apache/solr/bin/solr create_core -c appcatalog -force -/opt/apache/solr/bin/post -c appcatalog /tmp/samples.xml -if [ -d /etc/hadoop/conf ]; then - sed -i.bak 's/shared.loader=.*$/shared.loader=\/etc\/hadoop\/conf/g' /etc/tomcat/catalina.properties + +SOLR_OPTS=() + +if [ "${SOLR_STORAGE_TYPE}" == "hdfs" ]; then + SOLR_OPTS+=("-Dsolr.directoryFactory=HdfsDirectoryFactory") + SOLR_OPTS+=("-Dsolr.lock.type=hdfs") + if [ -e "$HADOOP_CONF_DIR" ]; then + SOLR_OPTS+=("-Dsolr.hdfs.confdir=${HADOOP_CONF_DIR}") + fi +fi + +if [ "${SOLR_DATA_DIR}" != "" ]; then + SOLR_OPTS+=("-Dsolr.data.dir=$SOLR_DATA_DIR") fi + if [ -e "$KEYTAB" ]; then + SOLR_OPTS+=("-Dsolr.hdfs.security.kerberos.enabled=true") + SOLR_OPTS+=("-Dsolr.hdfs.security.kerberos.keytabfile=${KEYTAB}") + SOLR_OPTS+=("-Dsolr.hdfs.security.kerberos.principal=${PRINCIPAL}") export JAVA_OPTS="$JAVA_OPTS -Djava.security.auth.login.config=/etc/tomcat/jaas.config -Djava.security.krb5.conf=/etc/krb5.conf -Djavax.security.auth.useSubjectCredsOnly=false" template_generator /etc/tomcat/jaas.config.template /etc/tomcat/jaas.config fi + +export SOLR_OPTS + +/opt/apache/solr/bin/solr start "${SOLR_OPTS[@]}" -p 8983 -force +/opt/apache/solr/bin/solr create_core -c appcatalog -force +/opt/apache/solr/bin/post -c appcatalog /tmp/samples.xml +if [ -d /etc/hadoop/conf ]; then + sed -i.bak 's/shared.loader=.*$/shared.loader=\/etc\/hadoop\/conf/g' /etc/tomcat/catalina.properties +fi + /usr/libexec/tomcat/server start diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md index e0d1c01..819b6f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md @@ -178,6 +178,60 @@ where `service-name` is user defined name. The deployment progress of the application catalog is located in Resource Manager UI. When the service reaches STABLE state, application catalog UI is available at: http://appcatalog.${SERVICE_NAME}.${USER}.${DOMAIN}:8080/ +Application Catalog environment options: + +| Environment variables | Description | +| KEYTAB | Service user keytab file for accessing HDFS. | +| PRINCIPAL | Service user Kerboers principal. | +| SOLR_DATA_DIR | Location to store Solr data. | +| SOLR_STORAGE_TYPE | Storage type for Solr data, supported type are: hdfs, local | +| SPNEGO_KEYTAB | Location of the keytab file used for authenticating HTTP endpoint. | +| SPNEGO_PRINCIPAL | The Kerberos principal to be used for HTTP endpoint. The principal MUST start with 'HTTP'/ as per Kerberos HTTP SPNEGO specification. | + +Secure application catalog Yarnfile example: +``` +{ + "name": "catalog", + "kerberos_principal" : { + "principal_name" : "catalog/_HOST@EXAMPLE.COM", + "keytab" : "file:///etc/security/keytabs/catalog.service.keytab" + }, + "version": "1", + "components" : + [ + { + "name": "appcatalog", + "number_of_containers": 1, + "artifact": { + "id": "apache/hadoop-yarn-applications-catalog-docker:3.3.0-SNAPSHOT", + "type": "DOCKER" + }, + "resource": { + "cpus": 1, + "memory": "256" + }, + "configuration": { + "env": { + "YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE":"true", + "YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS":"/etc/hadoop/conf:/etc/hadoop/conf:ro,/etc/krb5.conf:/etc/krb5.conf:ro,/etc/security/keytabs/catalog.service.keytab:/etc/security/keytabs/hbase.service.keytab:ro,/etc/security/keytabs/spnego.service.keytab:/etc/security/keytabs/spnego.service.keytab:ro", + "SPNEGO_KEYTAB":"/etc/security/keytabs/spnego.service.keytab", + "SPNEGO_PRINCIPAL":"HTTP/host-3.example.com@EXAMPLE.COM", + "KEYTAB":"/etc/security/keytabs/catalog.service.keytab", + "PRINCIPAL":"catalog/host3.example.com@EXAMPLE.COM", + "SOLR_DATA_DIR":"hdfs://host-1.example.com:9000/tmp/solr", + "SOLR_UPDATE_LOG":"hdfs://host-1.example.com:9000/tmp/solr", + "SOLR_STORAGE_TYPE":"hdfs" + }, + "properties": { + "docker.network": "host" + } + } + } + ] +} + +``` + ## Docker image ENTRYPOINT support Docker images may have built with ENTRYPOINT to enable start up of docker image without any parameters.