diff --git llap-server/pom.xml llap-server/pom.xml index 22ed693..97b22d5 100644 --- llap-server/pom.xml +++ llap-server/pom.xml @@ -229,6 +229,17 @@ ${basedir}/src/java ${basedir}/src/test + + + src/main/resources + + *.py + *.pyc + llap-daemon-log4j.properties + + false + + org.codehaus.mojo diff --git llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonConfiguration.java llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonConfiguration.java index 0554c32..35ebcc2 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonConfiguration.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonConfiguration.java @@ -42,13 +42,19 @@ public LlapDaemonConfiguration() { public static final String LLAP_DAEMON_YARN_SHUFFLE_PORT = LLAP_DAEMON_PREFIX + "yarn.shuffle.port"; public static final int LLAP_DAEMON_YARN_SHUFFLE_PORT_DEFAULT = 15551; - // Section for configs used in AM and executors public static final String LLAP_DAEMON_NUM_EXECUTORS = LLAP_DAEMON_PREFIX + "num.executors"; public static final int LLAP_DAEMON_NUM_EXECUTORS_DEFAULT = 4; public static final String LLAP_DAEMON_RPC_PORT = LLAP_DAEMON_PREFIX + "rpc.port"; public static final int LLAP_DAEMON_RPC_PORT_DEFAULT = 15001; + + public static final String LLAP_DAEMON_SERVICE_PORT = LLAP_DAEMON_PREFIX + "service.port"; + public static final int LLAP_DAEMON_SERVICE_PORT_DEFAULT = 15002; + + public static final String LLAP_DAEMON_SERVICE_SSL = LLAP_DAEMON_PREFIX + "service.ssl"; + public static final boolean LLAP_DAEMON_SERVICE_SSL_DEFAULT = false; + public static final String LLAP_DAEMON_MEMORY_PER_INSTANCE_MB = LLAP_DAEMON_PREFIX + "memory.per.instance.mb"; public static final int LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT = 4096; diff --git llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index 0ff255c..6581026 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.llap.daemon.LlapDaemonConfiguration; import org.apache.hadoop.hive.llap.daemon.registry.impl.LlapRegistryService; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos; +import org.apache.hadoop.hive.llap.daemon.services.impl.LlapWebServices; import org.apache.hadoop.hive.llap.io.api.LlapIoProxy; import org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorMetrics; import org.apache.hadoop.hive.llap.metrics.LlapMetricsSystem; @@ -51,6 +52,7 @@ private final LlapDaemonProtocolServerImpl server; private final ContainerRunnerImpl containerRunner; private final LlapRegistryService registry; + private final LlapWebServices webServices; private final AtomicLong numSubmissions = new AtomicLong(0); private JvmPauseMonitor pauseMonitor; private final ObjectName llapDaemonInfoBean; @@ -132,6 +134,7 @@ public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemor executorMemoryBytes, metrics); this.registry = new LlapRegistryService(); + this.webServices = new LlapWebServices(); } private void printAsciiArt() { @@ -153,6 +156,7 @@ public void serviceInit(Configuration conf) { server.init(conf); containerRunner.init(conf); registry.init(conf); + webServices.init(conf); LlapIoProxy.setDaemon(true); LlapIoProxy.initializeLlapIo(conf); } @@ -164,6 +168,7 @@ public void serviceStart() throws Exception { containerRunner.start(); registry.start(); registry.registerWorker(); + webServices.start(); } public void serviceStop() throws Exception { @@ -173,6 +178,7 @@ public void serviceStop() throws Exception { server.stop(); registry.unregisterWorker(); registry.stop(); + webServices.stop(); ShuffleHandler.shutdown(); } diff --git llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java index 33cd7eb..4f3841e 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java @@ -4,6 +4,10 @@ import java.net.Inet4Address; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; import java.net.UnknownHostException; import java.util.Map; @@ -23,6 +27,7 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.log4j.Logger; +import org.apache.tez.dag.api.TezUncheckedException; import org.apache.zookeeper.CreateMode; import com.google.common.base.Preconditions; @@ -100,6 +105,25 @@ public Endpoint getShuffleEndpoint() { return RegistryTypeUtils.inetAddrEndpoint("shuffle", ProtocolTypes.PROTOCOL_TCP, hostname, shufflePort); } + + public Endpoint getServicesEndpoint() { + final int servicePort = + conf.getInt(LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_PORT, + LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_PORT_DEFAULT); + final boolean isSSL = + conf.getBoolean(LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_SSL, + LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_SSL_DEFAULT); + final String scheme = isSSL ? "https" : "http"; + final URL serviceURL; + try { + serviceURL = new URL(scheme, hostname, servicePort, ""); + return RegistryTypeUtils.webEndpoint("services", serviceURL.toURI()); + } catch (MalformedURLException e) { + throw new TezUncheckedException(e); + } catch (URISyntaxException e) { + throw new TezUncheckedException("llap service URI for " + hostname + " is invalid", e); + } + } private final String getPath() { return RegistryPathUtils.join(RegistryUtils.componentPath(RegistryUtils.currentUser(), @@ -112,6 +136,7 @@ public void registerWorker() throws IOException { ServiceRecord srv = new ServiceRecord(); srv.addInternalEndpoint(getRpcEndpoint()); srv.addInternalEndpoint(getShuffleEndpoint()); + srv.addExternalEndpoint(getServicesEndpoint()); for (Map.Entry kv : this.conf) { if (kv.getKey().startsWith(LlapDaemonConfiguration.LLAP_DAEMON_PREFIX)) { diff --git llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebApp.java llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebApp.java new file mode 100644 index 0000000..cb57c9a --- /dev/null +++ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebApp.java @@ -0,0 +1,12 @@ +package org.apache.hadoop.hive.llap.daemon.services.impl; + +import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.YarnWebParams; + +public class LlapWebApp extends WebApp { + + @Override + public void setup() { + // JMX / config are defaults + } +} diff --git llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java new file mode 100644 index 0000000..2d1bbcb --- /dev/null +++ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java @@ -0,0 +1,53 @@ +package org.apache.hadoop.hive.llap.daemon.services.impl; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.llap.daemon.LlapDaemonConfiguration; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.WebApps; +import org.apache.log4j.Logger; + +public class LlapWebServices extends AbstractService { + + private static final Logger LOG = Logger.getLogger(LlapWebServices.class); + + private int port; + private boolean ssl; + private Configuration conf; + private WebApp webApp; + private LlapWebApp webAppInstance; + + public LlapWebServices() { + super("LlapWebServices"); + } + + @Override + public void serviceInit(Configuration conf) { + + this.conf = new Configuration(conf); + this.conf.addResource(YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); + + this.port = + conf.getInt(LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_PORT, + LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_PORT_DEFAULT); + this.ssl = + conf.getBoolean(LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_SSL, + LlapDaemonConfiguration.LLAP_DAEMON_SERVICE_SSL_DEFAULT); + + this.webAppInstance = new LlapWebApp(); + } + + @Override + public void serviceStart() throws Exception { + String bindAddress = "0.0.0.0"; + this.webApp = + WebApps.$for("llap").at(bindAddress).at(port).with(getConfig()) + /* TODO: security negotiation here */ + .start(); + } + + public void serviceStop() throws Exception { + this.webApp.stop(); + } +} diff --git llap-server/src/main/resources/webapps/llap/.keep llap-server/src/main/resources/webapps/llap/.keep new file mode 100644 index 0000000..e69de29