diff --git a/bin/hive b/bin/hive index 1ade51e..a7ae2f5 100755 --- a/bin/hive +++ b/bin/hive @@ -356,6 +356,7 @@ fi # include the log4j jar that is used for hive into the classpath CLASSPATH="${CLASSPATH}:${LOG_JAR_CLASSPATH}" export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${LOG_JAR_CLASSPATH}" +export JVM_PID="$$" if [ "$TORUN" = "" ] ; then echo "Service $SERVICE not found" diff --git a/common/src/java/org/apache/hive/common/util/ProcessUtils.java b/common/src/java/org/apache/hive/common/util/ProcessUtils.java new file mode 100644 index 0000000..a1be7da --- /dev/null +++ b/common/src/java/org/apache/hive/common/util/ProcessUtils.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.common.util; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Process related utilities. + */ +public class ProcessUtils { + private static Logger LOG = LoggerFactory.getLogger(ProcessUtils.class); + + public static String getPid() { + // JVM_PID is exported by bin/hive + String pidStr = System.getenv("JVM_PID"); + + // in case if it is not set correctly used fallback from mxbean which is implementation specific + if (pidStr == null || pidStr.trim().isEmpty()) { + String name = ManagementFactory.getRuntimeMXBean().getName(); + + if (name != null) { + int idx = name.indexOf("@"); + + if (idx != -1) { + String str = name.substring(0, name.indexOf("@")); + try { + Long.valueOf(str); + return str; + } catch (NumberFormatException nfe) { + throw new IllegalStateException("Process PID is not a number: " + str); + } + } + } + throw new IllegalStateException("Unsupported PID format: " + name); + } + return pidStr; + } + + public static int runCmdSync(List cmd) { + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Running command: " + cmd); + } + Process p = new ProcessBuilder(cmd).redirectErrorStream(true).start(); + return p.waitFor(); + } catch (InterruptedException | IOException e) { + return -1; + } + } +} diff --git a/common/src/java/org/apache/hive/http/HttpServer.java b/common/src/java/org/apache/hive/http/HttpServer.java index 3cb7a33..1e020ba 100644 --- a/common/src/java/org/apache/hive/http/HttpServer.java +++ b/common/src/java/org/apache/hive/http/HttpServer.java @@ -558,6 +558,12 @@ private void initializeWebServer(final Builder b, int queueSize) throws IOExcept addServlet("conf", "/conf", ConfServlet.class); addServlet("stacks", "/stacks", StackServlet.class); addServlet("conflog", "/conflog", Log4j2ConfiguratorServlet.class); + final String asyncProfilerHome = ProfileServlet.getAsyncProfilerHome(); + if (asyncProfilerHome != null && !asyncProfilerHome.trim().isEmpty()) { + addServlet("prof", "/prof", ProfileServlet.class); + } else { + LOG.info("ASYNC_PROFILER_HOME env or -Dasync.profiler.home not specified. Disabling /prof endpoint.."); + } for (Pair> p : b.servlets) { addServlet(p.getFirst(), "/" + p.getFirst(), p.getSecond()); diff --git a/common/src/java/org/apache/hive/http/ProfileServlet.java b/common/src/java/org/apache/hive/http/ProfileServlet.java new file mode 100644 index 0000000..a6ce7f0 --- /dev/null +++ b/common/src/java/org/apache/hive/http/ProfileServlet.java @@ -0,0 +1,374 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.http; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.hive.common.util.ProcessUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Servlet that runs async-profiler as web-endpoint. + * Following options from async-profiler can be specified as query paramater. + * // -e event profiling event: cpu|alloc|lock|cache-misses etc. + * // -d duration run profiling for seconds (integer) + * // -i interval sampling interval in nanoseconds (long) + * // -j jstackdepth maximum Java stack depth (integer) + * // -b bufsize frame buffer size (long) + * // -t profile different threads separately + * // -s simple class names instead of FQN + * // -o fmt[,fmt...] output format: summary|traces|flat|collapsed|svg|tree|jfr + * // --width px SVG width pixels (integer) + * // --height px SVG frame height pixels (integer) + * // --minwidth px skip frames smaller than px (double) + * // --reverse generate stack-reversed FlameGraph / Call tree + * Example: + * - To collect 30 second CPU profile of current process (returns FlameGraph svg) + * curl "http://localhost:10002/prof" + * - To collect 1 minute CPU profile of current process and output in tree format (html) + * curl "http://localhost:10002/prof?output=tree&duration=60" + * - To collect 30 second heap allocation profile of current process (returns FlameGraph svg) + * curl "http://localhost:10002/prof?event=alloc" + * - To collect lock contention profile of current process (returns FlameGraph svg) + * curl "http://localhost:10002/prof?event=lock" + * Following event types are supported (default is 'cpu') (NOTE: not all OS'es support all events) + * // Perf events: + * // cpu + * // page-faults + * // context-switches + * // cycles + * // instructions + * // cache-references + * // cache-misses + * // branches + * // branch-misses + * // bus-cycles + * // L1-dcache-load-misses + * // LLC-load-misses + * // dTLB-load-misses + * // mem:breakpoint + * // trace:tracepoint + * // Java events: + * // alloc + * // lock + */ +public class ProfileServlet extends HttpServlet { + private static final long serialVersionUID = 1L; + private static final Logger LOG = LoggerFactory.getLogger(ProfileServlet.class); + private static final String ACCESS_CONTROL_ALLOW_METHODS = "Access-Control-Allow-Methods"; + private static final String ALLOWED_METHODS = "GET"; + private static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin"; + private static final String CONTENT_TYPE_SVG = "image/svg+xml; charset=utf-8"; + private static final String CONTENT_TYPE_HTML = "text/html; charset=utf-8"; + private static final String CONTENT_TYPE_TEXT = "text/plain; charset=utf-8"; + private static final String CONTENT_TYPE_BINARY = "application/octet-stream"; + private static final String ASYNC_PROFILER_HOME_ENV = "ASYNC_PROFILER_HOME"; + private static final String ASYNC_PROFILER_HOME_SYSTEM_PROPERTY = "async.profiler.home"; + private static final String PROFILER_SCRIPT = "/profiler.sh"; + private static final Output DEFAULT_OUTPUT_TYPE = Output.SVG; + private static final int DEFAULT_DURATION_SECONDS = 30; + + enum Event { + CPU("cpu"), + ALLOC("alloc"), + LOCK("lock"), + PAGE_FAULTS("page-faults"), + CONTEXT_SWITCHES("context-switches"), + CYCLES("cycles"), + INSTRUCTIONS("instructions"), + CACHE_REFERENCES("cache-references"), + CACHE_MISSES("cache-misses"), + BRANCHES("branches"), + BRANCH_MISSES("branch-misses"), + BUS_CYCLES("bus-cycles"), + L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"), + LLC_LOAD_MISSES("LLC-load-misses"), + DTLB_LOAD_MISSES("dTLB-load-misses"), + MEM_BREAKPOINT("mem:breakpoint"), + TRACE_TRACEPOINT("trace:tracepoint"),; + + private String internalName; + + Event(final String internalName) { + this.internalName = internalName; + } + + public String getInternalName() { + return internalName; + } + + public static Event fromInternalName(final String name) { + for (Event event : values()) { + if (event.getInternalName().equalsIgnoreCase(name)) { + return event; + } + } + + return null; + } + } + + enum Output { + SUMMARY, + TRACES, + FLAT, + COLLAPSED, + SVG, + TREE, + JFR + } + + private Lock profilerLock = new ReentrantLock(); + private String pid; + private String asyncProfilerHome; + + public ProfileServlet() { + this.asyncProfilerHome = getAsyncProfilerHome(); + try { + this.pid = ProcessUtils.getPid(); + } catch (IllegalStateException e) { + this.pid = null; + } + LOG.info("Servlet process PID: {} asyncProfilerHome: {}", pid, asyncProfilerHome); + } + + @Override + protected void doGet(final HttpServletRequest req, final HttpServletResponse resp) throws IOException { + if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(), req, resp)) { + resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED); + setResponseHeader(resp); + resp.getWriter().write("Unauthorized: Instrumentation access is not allowed!"); + return; + } + + // make sure async profiler home is set + if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) { + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + setResponseHeader(resp); + resp.getWriter().write("ASYNC_PROFILER_HOME env is not set."); + return; + } + + // if pid is explicitly specified, use it else default to current process + if (req.getParameter("pid") != null) { + pid = req.getParameter("pid"); + } + + // if pid is not specified in query param and if current process pid cannot be determined + if (pid == null || pid.trim().isEmpty()) { + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + setResponseHeader(resp); + resp.getWriter().write("'pid' query parameter unspecified or unable to determine PID of current process."); + return; + } + + // 30s default duration + int duration = DEFAULT_DURATION_SECONDS; + if (req.getParameter("duration") != null) { + try { + duration = Integer.parseInt(req.getParameter("duration")); + } catch (NumberFormatException e) { + // ignore and use default + } + } + + // default to svg flamegraph + Output output = DEFAULT_OUTPUT_TYPE; + if (req.getParameter("output") != null) { + output = Output.valueOf(req.getParameter("output").trim().toUpperCase()); + } + + final Event event = getEvent(req); + final Long interval = getLong(req, "interval"); + final Integer jstackDepth = getInteger(req, "jstackdepth"); + final Long bufsize = getLong(req, "bufsize"); + final String thread = req.getParameter("thread"); + final String simple = req.getParameter("simple"); + final Integer width = getInteger(req, "width"); + final Integer height = getInteger(req, "height"); + final Double minwidth = getMinWidth(req); + final String reverse = req.getParameter("reverse"); + + try { + int lockTimeoutSecs = 5; + if (profilerLock.tryLock(lockTimeoutSecs, TimeUnit.SECONDS)) { + try { + // profiles are stored in temp dir and deleted on jvm exit + File outputFile = File.createTempFile("async-prof-pid-" + pid, "." + output.name().toLowerCase()); + outputFile.deleteOnExit(); + List cmd = new ArrayList<>(); + cmd.add(asyncProfilerHome + PROFILER_SCRIPT); + if (event != null) { + cmd.add("-e"); + cmd.add(event.getInternalName()); + } + cmd.add("-d"); + cmd.add("" + duration); + cmd.add("-o"); + cmd.add(output.name().toLowerCase()); + cmd.add("-f"); + cmd.add(outputFile.getAbsolutePath()); + if (interval != null) { + cmd.add("-i"); + cmd.add(interval.toString()); + } + if (jstackDepth != null) { + cmd.add("-j"); + cmd.add(jstackDepth.toString()); + } + if (bufsize != null) { + cmd.add("-b"); + cmd.add(bufsize.toString()); + } + if (thread != null) { + cmd.add("-t"); + } + if (simple != null) { + cmd.add("-s"); + } + if (width != null) { + cmd.add("--width"); + cmd.add(width.toString()); + } + if (height != null) { + cmd.add("--height"); + cmd.add(height.toString()); + } + if (minwidth != null) { + cmd.add("--minwidth"); + cmd.add(minwidth.toString()); + } + if (reverse != null) { + cmd.add("--reverse"); + } + cmd.add(pid); + int ret = ProcessUtils.runCmdSync(cmd); + if (ret == 0) { + setResponseHeader(resp, output); + byte[] b = Files.readAllBytes(outputFile.toPath()); + OutputStream os = resp.getOutputStream(); + os.write(b); + os.flush(); + } else { + setResponseHeader(resp); + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + resp.getWriter().write("Error executing async-profiler"); + } + } finally { + profilerLock.unlock(); + } + } else { + setResponseHeader(resp); + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + resp.getWriter().write("Unable to acquire lock. Another instance of profiler might be running."); + LOG.warn("Unable to acquire lock in {} seconds. Another instance of profiler might be running.", + lockTimeoutSecs); + } + } catch (InterruptedException e) { + LOG.warn("Interrupted while acquiring profile lock.", e); + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + } + + private Integer getInteger(final HttpServletRequest req, final String param) { + final String value = req.getParameter(param); + if (value != null) { + try { + return Integer.valueOf(value); + } catch (NumberFormatException e) { + return null; + } + } + return null; + } + + private Long getLong(final HttpServletRequest req, final String param) { + final String value = req.getParameter(param); + if (value != null) { + try { + return Long.valueOf(value); + } catch (NumberFormatException e) { + return null; + } + } + return null; + } + + private Double getMinWidth(final HttpServletRequest req) { + final String value = req.getParameter("minwidth"); + if (value != null) { + try { + return Double.valueOf(value); + } catch (NumberFormatException e) { + return null; + } + } + return null; + } + + private Event getEvent(final HttpServletRequest req) { + final String eventArg = req.getParameter("event"); + if (eventArg != null) { + return Event.fromInternalName(eventArg); + } + return null; + } + + private void setResponseHeader(final HttpServletResponse response) { + setResponseHeader(response, null); + } + + private void setResponseHeader(final HttpServletResponse response, final Output output) { + response.setHeader(ACCESS_CONTROL_ALLOW_METHODS, ALLOWED_METHODS); + response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*"); + if (output == null) { + response.setContentType(CONTENT_TYPE_TEXT); + } else { + if (output.equals(Output.SVG)) { + response.setContentType(CONTENT_TYPE_SVG); + } else if (output.equals(Output.TREE)) { + response.setContentType(CONTENT_TYPE_HTML); + } else if (output.equals(Output.JFR)) { + response.setContentType(CONTENT_TYPE_BINARY); + } else { + response.setContentType(CONTENT_TYPE_TEXT); + } + } + } + + static String getAsyncProfilerHome() { + String asyncProfilerHome = System.getenv(ASYNC_PROFILER_HOME_ENV); + // if ENV is not set, see if -Dasync.profiler.home=/path/to/asyn/profiler/home is set + if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) { + asyncProfilerHome = System.getProperty(ASYNC_PROFILER_HOME_SYSTEM_PROPERTY); + } + + return asyncProfilerHome; + } +}