diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapDaemonInfo.java b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapDaemonInfo.java index fa29b59..dae10c8 100644 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapDaemonInfo.java +++ b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapDaemonInfo.java @@ -25,12 +25,13 @@ private static final class LlapDaemonInfoHolder { public LlapDaemonInfoHolder(int numExecutors, long executorMemory, long cacheSize, - boolean isDirectCache, boolean isLlapIo) { + boolean isDirectCache, boolean isLlapIo, final String pid) { this.numExecutors = numExecutors; this.executorMemory = executorMemory; this.cacheSize = cacheSize; this.isDirectCache = isDirectCache; this.isLlapIo = isLlapIo; + this.PID = pid; } final int numExecutors; @@ -38,6 +39,7 @@ public LlapDaemonInfoHolder(int numExecutors, long executorMemory, long cacheSiz final long cacheSize; final boolean isDirectCache; final boolean isLlapIo; + final String PID; } // add more variables as required @@ -51,13 +53,14 @@ public static void initialize(String appName, Configuration daemonConf) { long ioMemoryBytes = HiveConf.getSizeVar(daemonConf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE); boolean isDirectCache = HiveConf.getBoolVar(daemonConf, ConfVars.LLAP_ALLOCATOR_DIRECT); boolean isLlapIo = HiveConf.getBoolVar(daemonConf, HiveConf.ConfVars.LLAP_IO_ENABLED, true); - initialize(appName, numExecutors, executorMemoryBytes, ioMemoryBytes, isDirectCache, isLlapIo); + String pid = System.getenv("JVM_PID"); + initialize(appName, numExecutors, executorMemoryBytes, ioMemoryBytes, isDirectCache, isLlapIo, pid); } public static void initialize(String appName, int numExecutors, long executorMemoryBytes, - long ioMemoryBytes, boolean isDirectCache, boolean isLlapIo) { + long ioMemoryBytes, boolean isDirectCache, boolean isLlapIo, final String pid) { INSTANCE.dataRef.set(new LlapDaemonInfoHolder(numExecutors, executorMemoryBytes, ioMemoryBytes, - isDirectCache, isLlapIo)); + isDirectCache, isLlapIo, pid)); } public boolean isLlap() { @@ -89,4 +92,7 @@ public boolean isLlapIo() { return dataRef.get().isLlapIo; } + public String getPID() { + return dataRef.get().PID; + } } diff --git a/llap-server/bin/runLlapDaemon.sh b/llap-server/bin/runLlapDaemon.sh index 82c2cc5..5a0c10e 100755 --- a/llap-server/bin/runLlapDaemon.sh +++ b/llap-server/bin/runLlapDaemon.sh @@ -127,6 +127,6 @@ LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS} -Dllap.daemon.log.file=${LLAP_DAEMON_LOG_F LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS} -Dllap.daemon.root.logger=${LLAP_DAEMON_LOGGER}" LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS} -Dllap.daemon.log.level=${LLAP_DAEMON_LOG_LEVEL}" +export JVM_PID="$$" exec "$JAVA" -Dproc_llapdaemon -Xms${LLAP_DAEMON_HEAPSIZE}m -Xmx${LLAP_DAEMON_HEAPSIZE}m ${LLAP_DAEMON_OPTS} -classpath "$CLASSPATH" $CLASS "$@" - diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index aae146e..db309f3 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -14,8 +14,6 @@ package org.apache.hadoop.hive.llap.daemon.impl; -import org.apache.hadoop.hive.llap.LlapOutputFormatService; - import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.MemoryPoolMXBean; @@ -42,6 +40,7 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.DaemonId; import org.apache.hadoop.hive.llap.LlapDaemonInfo; +import org.apache.hadoop.hive.llap.LlapOutputFormatService; import org.apache.hadoop.hive.llap.LlapUtil; import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration; import org.apache.hadoop.hive.llap.daemon.ContainerRunner; @@ -90,7 +89,6 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, LlapDaemonMXBean { private static final Logger LOG = LoggerFactory.getLogger(LlapDaemon.class); - private final Configuration shuffleHandlerConf; private final SecretManager secretManager; private final LlapProtocolServerImpl server; @@ -243,7 +241,7 @@ public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemor pauseMonitor.start(); String displayNameJvm = "LlapDaemonJvmMetrics-" + hostName; String sessionId = MetricsUtils.getUUID(); - LlapDaemonJvmMetrics.create(displayNameJvm, sessionId); + LlapDaemonJvmMetrics.create(displayNameJvm, sessionId, daemonConf); String displayName = "LlapDaemonExecutorMetrics-" + hostName; daemonConf.set("llap.daemon.metrics.sessionid", sessionId); String[] strIntervals = HiveConf.getTrimmedStringsVar(daemonConf, @@ -537,7 +535,7 @@ public static void main(String[] args) throws Exception { llapDaemon.init(daemonConf); llapDaemon.start(); - LOG.info("Started LlapDaemon"); + LOG.info("Started LlapDaemon with PID: {}", LlapDaemonInfo.INSTANCE.getPID()); // Relying on the RPC threads to keep the service alive. } catch (Throwable t) { // TODO Replace this with a ExceptionHandler / ShutdownHook diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmInfo.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmInfo.java index efbddaa..11ac5a4 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmInfo.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmInfo.java @@ -38,6 +38,8 @@ LlapDaemonMappedBufferMemoryUsed("Estimate of memory that JVM is using for mapped byte buffers in bytes"), LlapDaemonOpenFileDescriptorCount("Number of open file descriptors"), LlapDaemonMaxFileDescriptorCount("Maximum number of file descriptors"), + LlapDaemonResidentSetSize("Resident memory (RSS) used by llap daemon process in bytes"), + LlapDaemonVirtualMemorySize("Virtual memory (VMEM) used by llap daemon process in bytes") ; private final String desc; diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmMetrics.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmMetrics.java index cfb8729..be25846 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmMetrics.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmMetrics.java @@ -26,6 +26,8 @@ import static org.apache.hadoop.hive.llap.metrics.LlapDaemonJvmInfo.LlapDaemonMappedBufferTotalCapacity; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonJvmInfo.LlapDaemonMaxFileDescriptorCount; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonJvmInfo.LlapDaemonOpenFileDescriptorCount; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonJvmInfo.LlapDaemonResidentSetSize; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonJvmInfo.LlapDaemonVirtualMemorySize; import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName; import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId; @@ -34,12 +36,17 @@ import java.lang.management.OperatingSystemMXBean; import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.llap.LlapDaemonInfo; +import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsSource; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.MetricsRegistry; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; import com.sun.management.UnixOperatingSystemMXBean; @@ -51,24 +58,34 @@ private final String name; private final String sessionId; private final MetricsRegistry registry; + private final ResourceCalculatorProcessTree processTree; + private final String daemonPid = LlapDaemonInfo.INSTANCE.getPID(); - private LlapDaemonJvmMetrics(String displayName, String sessionId) { + private LlapDaemonJvmMetrics(String displayName, String sessionId, final Configuration conf) { this.name = displayName; this.sessionId = sessionId; + Class clazz = conf.getClass(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, + null, ResourceCalculatorProcessTree.class); + this.processTree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("" + daemonPid, clazz, conf); + if (processTree != null) { + this.processTree.setConf(conf); + } this.registry = new MetricsRegistry("LlapDaemonJvmRegistry"); this.registry.tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME).tag(SessionId, sessionId); } - public static LlapDaemonJvmMetrics create(String displayName, String sessionId) { + public static LlapDaemonJvmMetrics create(String displayName, String sessionId, + final Configuration conf) { MetricsSystem ms = LlapMetricsSystem.instance(); - return ms.register(displayName, "LlapDaemon JVM Metrics", new LlapDaemonJvmMetrics(displayName, sessionId)); + return ms.register(displayName, "LlapDaemon JVM Metrics", + new LlapDaemonJvmMetrics(displayName, sessionId, conf)); } @Override public void getMetrics(MetricsCollector collector, boolean b) { MetricsRecordBuilder rb = collector.addRecord(LlapDaemonJVMMetrics) .setContext("jvm") - .tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME) + .tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME + "(PID: " + daemonPid + ")") .tag(SessionId, sessionId); getJvmMetrics(rb); } @@ -100,6 +117,12 @@ private void getJvmMetrics(final MetricsRecordBuilder rb) { openFileHandles = ((UnixOperatingSystemMXBean) os).getOpenFileDescriptorCount(); maxFileHandles = ((UnixOperatingSystemMXBean) os).getMaxFileDescriptorCount(); } + long rss = 0; + long vmem = 0; + if (processTree != null) { + rss = processTree.getRssMemorySize(); + vmem = processTree.getVirtualMemorySize(); + } rb.addGauge(LlapDaemonDirectBufferCount, directBufferCount) .addGauge(LlapDaemonDirectBufferTotalCapacity, directBufferTotalCapacity) .addGauge(LlapDaemonDirectBufferMemoryUsed, directBufferMemoryUsed) @@ -107,7 +130,9 @@ private void getJvmMetrics(final MetricsRecordBuilder rb) { .addGauge(LlapDaemonMappedBufferTotalCapacity, mappedBufferTotalCapacity) .addGauge(LlapDaemonMappedBufferMemoryUsed, mappedBufferMemoryUsed) .addGauge(LlapDaemonOpenFileDescriptorCount, openFileHandles) - .addGauge(LlapDaemonMaxFileDescriptorCount, maxFileHandles); + .addGauge(LlapDaemonMaxFileDescriptorCount, maxFileHandles) + .addGauge(LlapDaemonResidentSetSize, rss) + .addGauge(LlapDaemonVirtualMemorySize, vmem); } public String getName() { diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java index 6f1305e..6af230e 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java @@ -121,7 +121,7 @@ private MiniLlapCluster(String clusterName, @Nullable MiniZooKeeperCluster miniZ this.ioBytesPerService = ioBytesPerService; LlapDaemonInfo.initialize("mini-llap-cluster", numExecutorsPerService, execMemoryPerService, - ioBytesPerService, ioIsDirect, llapIoEnabled); + ioBytesPerService, ioIsDirect, llapIoEnabled, "-1"); // Setup Local Dirs localDirs = new String[numLocalDirs];