diff --git a/bin/hive b/bin/hive index 1ade51e..a7ae2f5 100755 --- a/bin/hive +++ b/bin/hive @@ -356,6 +356,7 @@ fi # include the log4j jar that is used for hive into the classpath CLASSPATH="${CLASSPATH}:${LOG_JAR_CLASSPATH}" export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${LOG_JAR_CLASSPATH}" +export JVM_PID="$$" if [ "$TORUN" = "" ] ; then echo "Service $SERVICE not found" diff --git a/common/src/java/org/apache/hive/common/util/ProcessUtils.java b/common/src/java/org/apache/hive/common/util/ProcessUtils.java new file mode 100644 index 0000000..3489fc4 --- /dev/null +++ b/common/src/java/org/apache/hive/common/util/ProcessUtils.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.common.util; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Process related utilities. + */ +public class ProcessUtils { + private static Logger LOG = LoggerFactory.getLogger(ProcessUtils.class); + + public static String getPid() { + // JVM_PID is exported by bin/ext/util/execHiveCmd.sh + String pidStr = System.getenv("JVM_PID"); + + // in case if it is not set correctly used fallback from mxbean which is implementation specific + if (pidStr == null || pidStr.trim().isEmpty()) { + String name = ManagementFactory.getRuntimeMXBean().getName(); + + if (name != null) { + int idx = name.indexOf("@"); + + if (idx != -1) { + String str = name.substring(0, name.indexOf("@")); + try { + Long.valueOf(str); + return str; + } catch (NumberFormatException nfe) { + throw new IllegalStateException("Process PID is not a number: " + str); + } + } + } + throw new IllegalStateException("Unsupported PID format: " + name); + } + return pidStr; + } + + public static int runCmdSync(List cmd) { + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Running command: " + cmd); + } + Process p = new ProcessBuilder(cmd).redirectErrorStream(true).start(); + return p.waitFor(); + } catch (InterruptedException | IOException e) { + return -1; + } + } +} diff --git a/common/src/java/org/apache/hive/http/HttpServer.java b/common/src/java/org/apache/hive/http/HttpServer.java index 3cb7a33..de170b7 100644 --- a/common/src/java/org/apache/hive/http/HttpServer.java +++ b/common/src/java/org/apache/hive/http/HttpServer.java @@ -558,6 +558,7 @@ private void initializeWebServer(final Builder b, int queueSize) throws IOExcept addServlet("conf", "/conf", ConfServlet.class); addServlet("stacks", "/stacks", StackServlet.class); addServlet("conflog", "/conflog", Log4j2ConfiguratorServlet.class); + addServlet("prof", "/prof", ProfileServlet.class); for (Pair> p : b.servlets) { addServlet(p.getFirst(), "/" + p.getFirst(), p.getSecond()); diff --git a/common/src/java/org/apache/hive/http/ProfileServlet.java b/common/src/java/org/apache/hive/http/ProfileServlet.java new file mode 100644 index 0000000..debefcf --- /dev/null +++ b/common/src/java/org/apache/hive/http/ProfileServlet.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.http; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.hive.common.util.ProcessUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Servlet that runs async-profiler as web-endpoint. + * + * Following options from async-profiler can be specified as query paramater. + * // -e event profiling event: cpu|alloc|lock|cache-misses etc. + * // -d duration run profiling for seconds + * // -i interval sampling interval in nanoseconds + * // -j jstackdepth maximum Java stack depth + * // -b bufsize frame buffer size + * // -t profile different threads separately + * // -s simple class names instead of FQN + * // -o fmt[,fmt...] output format: summary|traces|flat|collapsed|svg|tree|jfr + * // --title string SVG title + * // --width px SVG width + * // --height px SVG frame height + * // --minwidth px skip frames smaller than px + * // --reverse generate stack-reversed FlameGraph / Call tree + * + * Example: + * + * - To collect 30 second CPU profile of current process (returns FlameGraph svg) + * + * curl "http://localhost:10002/prof" + * + * - To collect 1 minute CPU profile of current process and output in tree format (html) + * + * curl "http://localhost:10002/prof?output=tree&duration=60" + * + * - To collect 30 second heap allocation profile of current process (returns FlameGraph svg) + * + * curl "http://localhost:10002/prof?event=alloc" + * + * - To collect lock contention profile of current process (returns FlameGraph svg) + * + * curl "http://localhost:10002/prof?event=lock" + * + * Following event types are supported (default is 'cpu') (NOTE: not all OS'es support all events) + * // Perf events: + * // cpu + * // page-faults + * // context-switches + * // cycles + * // instructions + * // cache-references + * // cache-misses + * // branches + * // branch-misses + * // bus-cycles + * // L1-dcache-load-misses + * // LLC-load-misses + * // dTLB-load-misses + * // mem:breakpoint + * // trace:tracepoint + * // Java events: + * // alloc + * // lock + */ +public class ProfileServlet extends HttpServlet { + private static final long serialVersionUID = 1L; + private static final Logger LOG = LoggerFactory.getLogger(ProfileServlet.class); + private static final String ACCESS_CONTROL_ALLOW_METHODS = "Access-Control-Allow-Methods"; + private static final String ALLOWED_METHODS = "GET"; + private static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin"; + private static final String CONTENT_TYPE_SVG = "image/svg+xml; charset=utf-8"; + private static final String CONTENT_TYPE_HTML = "text/html; charset=utf-8"; + private static final String CONTENT_TYPE_TEXT = "text/plain; charset=utf-8"; + private static final String CONTENT_TYPE_BINARY = "application/octet-stream"; + private static final String ASYNC_PROFILER_HOME_ENV = "ASYNC_PROFILER_HOME"; + private static final String ASYNC_PROFILER_HOME_SYSTEM_PROPERTY = "async.profiler.home"; + private static final String PROFILER_SCRIPT = "/profiler.sh"; + private static final String DEFAULT_OUTPUT_TYPE = "svg"; + private static final int DEFAULT_DURATION_SECONDS = 30; + + private Lock profilerLock = new ReentrantLock(); + private String pid; + private String asyncProfilerHome; + + public ProfileServlet() { + this.asyncProfilerHome = System.getenv(ASYNC_PROFILER_HOME_ENV); + // if ENV is not set, see if -Dasync.profiler.home=/path/to/asyn/profiler/home is set + if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) { + this.asyncProfilerHome = System.getProperty(ASYNC_PROFILER_HOME_SYSTEM_PROPERTY); + } + try { + this.pid = ProcessUtils.getPid(); + } catch (IllegalStateException e) { + this.pid = null; + } + LOG.info("Servlet process PID: {} asyncProfilerHome: {}", pid, asyncProfilerHome); + } + + @Override + protected void doGet(final HttpServletRequest req, final HttpServletResponse resp) throws IOException { + if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(), req, resp)) { + resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED); + setResponseHeader(resp, "text"); + resp.getWriter().write("Unauthorized: Instrumentation access is not allowed!"); + return; + } + + // make sure async profiler home is set + if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) { + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + setResponseHeader(resp, "text"); + resp.getWriter().write("ASYNC_PROFILER_HOME env is not set."); + return; + } + + // if pid is explicitly specified, use it else default to current process + if (req.getParameter("pid") != null) { + pid = req.getParameter("pid"); + } + + // if pid is not specified in query param and if current process pid cannot be determined + if (pid == null || pid.trim().isEmpty()) { + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + setResponseHeader(resp, "text"); + resp.getWriter().write("'pid' query parameter unspecified or unable to determine PID of current process."); + return; + } + + // 30s default duration + int duration = DEFAULT_DURATION_SECONDS; + if (req.getParameter("duration") != null) { + try { + duration = Integer.parseInt(req.getParameter("duration")); + } catch (NumberFormatException e) { + // ignore and use default + } + } + + // default to svg flamegraph + String output = DEFAULT_OUTPUT_TYPE; + if (req.getParameter("output") != null) { + output = req.getParameter("output").trim().toLowerCase(); + } + + final String event = req.getParameter("event"); + final String interval = req.getParameter("interval"); + final String jstackDepth = req.getParameter("jstackdepth"); + final String bufsize = req.getParameter("bufsize"); + final String thread = req.getParameter("thread"); + final String simple = req.getParameter("simple"); + final String title = req.getParameter("title"); + final String width = req.getParameter("width"); + final String height = req.getParameter("height"); + final String minwidth = req.getParameter("minwidth"); + final String reverse = req.getParameter("reverse"); + + profilerLock.lock(); + try { + // profiles are stored in temp dir and deleted on jvm exit + File outputFile = File.createTempFile("async-prof-pid-" + pid, "." + output); + outputFile.deleteOnExit(); + List cmd = new ArrayList<>(); + cmd.add(asyncProfilerHome + PROFILER_SCRIPT); + if (event != null) { + cmd.add("-e"); + cmd.add(event); + } + cmd.add("-d"); + cmd.add("" + duration); + cmd.add("-o"); + cmd.add(output); + cmd.add("-f"); + cmd.add(outputFile.getAbsolutePath()); + if (interval != null) { + cmd.add("-i"); + cmd.add(interval); + } + if (jstackDepth != null) { + cmd.add("-j"); + cmd.add(jstackDepth); + } + if (bufsize != null) { + cmd.add("-b"); + cmd.add(bufsize); + } + if (thread != null) { + cmd.add("-t"); + } + if (simple != null) { + cmd.add("-s"); + } + if (title != null) { + cmd.add("--title"); + cmd.add(title); + } + if (width != null) { + cmd.add("--width"); + cmd.add(width); + } + if (height != null) { + cmd.add("--height"); + cmd.add(height); + } + if (minwidth != null) { + cmd.add("--minwidth"); + cmd.add(minwidth); + } + if (reverse != null) { + cmd.add("--reverse"); + } + cmd.add(pid); + int ret = ProcessUtils.runCmdSync(cmd); + if (ret == 0) { + setResponseHeader(resp, output); + byte[] b = Files.readAllBytes(outputFile.toPath()); + OutputStream os = resp.getOutputStream(); + os.write(b); + os.flush(); + } else { + setResponseHeader(resp, "text"); + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + resp.getWriter().write("Error executing async-profiler"); + } + } finally { + profilerLock.unlock(); + } + } + + private void setResponseHeader(final HttpServletResponse response, final String output) { + response.setHeader(ACCESS_CONTROL_ALLOW_METHODS, ALLOWED_METHODS); + response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*"); + if (output.equalsIgnoreCase("svg")) { + response.setContentType(CONTENT_TYPE_SVG); + } else if (output.equalsIgnoreCase("tree")) { + response.setContentType(CONTENT_TYPE_HTML); + } else if (output.equalsIgnoreCase("jfr")) { + response.setContentType(CONTENT_TYPE_BINARY); + } else { + response.setContentType(CONTENT_TYPE_TEXT); + } + } +}