diff --git a/llap-server/bin/runLlapDaemon.sh b/llap-server/bin/runLlapDaemon.sh index f4e9ddd..82c2cc5 100755 --- a/llap-server/bin/runLlapDaemon.sh +++ b/llap-server/bin/runLlapDaemon.sh @@ -107,7 +107,7 @@ elif [ "$COMMAND" = "run" ] ; then CLASS='org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon' fi -JAVA_OPTS_BASE="${JAVA_OPTS_BASE} -Xloggc:${LLAP_DAEMON_LOG_DIR}/gc.log" +JAVA_OPTS_BASE="${JAVA_OPTS_BASE} -Xloggc:${LLAP_DAEMON_LOG_DIR}/gc_$(date +%Y-%m-%d-%H).log" LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS} ${JAVA_OPTS_BASE}" # Set the default GC option if none set diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index 30f9ad4..3080c39 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -21,8 +21,10 @@ import java.lang.management.MemoryType; import java.net.InetSocketAddress; import java.net.URL; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; @@ -173,28 +175,33 @@ public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemor daemonConf, ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE); boolean enablePreemption = HiveConf.getBoolVar( daemonConf, ConfVars.LLAP_DAEMON_TASK_SCHEDULER_ENABLE_PREEMPTION); - LOG.warn("Attempting to start LlapDaemonConf with the following configuration: " + - "maxJvmMemory=" + maxJvmMemory + " (" - + LlapUtil.humanReadableByteCount(maxJvmMemory) + ")" + - ", requestedExecutorMemory=" + executorMemoryBytes + - " (" + LlapUtil.humanReadableByteCount(executorMemoryBytes) + ")" + - ", llapIoCacheSize=" + ioMemoryBytes + " (" - + LlapUtil.humanReadableByteCount(ioMemoryBytes) + ")" + - ", xmxHeadRoomMemory=" + xmxHeadRoomBytes + " (" - + LlapUtil.humanReadableByteCount(xmxHeadRoomBytes) + ")" + - ", adjustedExecutorMemory=" + executorMemoryPerInstance + - " (" + LlapUtil.humanReadableByteCount(executorMemoryPerInstance) + ")" + - ", numExecutors=" + numExecutors + - ", llapIoEnabled=" + ioEnabled + - ", llapIoCacheIsDirect=" + isDirectCache + - ", rpcListenerPort=" + srvPort + - ", mngListenerPort=" + mngPort + - ", webPort=" + webPort + - ", outputFormatSvcPort=" + outputFormatServicePort + - ", workDirs=" + Arrays.toString(localDirs) + - ", shufflePort=" + shufflePort + - ", waitQueueSize= " + waitQueueSize + - ", enablePreemption= " + enablePreemption); + final String logMsg = "Attempting to start LlapDaemon with the following configuration: " + + "maxJvmMemory=" + maxJvmMemory + " (" + + LlapUtil.humanReadableByteCount(maxJvmMemory) + ")" + + ", requestedExecutorMemory=" + executorMemoryBytes + + " (" + LlapUtil.humanReadableByteCount(executorMemoryBytes) + ")" + + ", llapIoCacheSize=" + ioMemoryBytes + " (" + + LlapUtil.humanReadableByteCount(ioMemoryBytes) + ")" + + ", xmxHeadRoomMemory=" + xmxHeadRoomBytes + " (" + + LlapUtil.humanReadableByteCount(xmxHeadRoomBytes) + ")" + + ", adjustedExecutorMemory=" + executorMemoryPerInstance + + " (" + LlapUtil.humanReadableByteCount(executorMemoryPerInstance) + ")" + + ", numExecutors=" + numExecutors + + ", llapIoEnabled=" + ioEnabled + + ", llapIoCacheIsDirect=" + isDirectCache + + ", rpcListenerPort=" + srvPort + + ", mngListenerPort=" + mngPort + + ", webPort=" + webPort + + ", outputFormatSvcPort=" + outputFormatServicePort + + ", workDirs=" + Arrays.toString(localDirs) + + ", shufflePort=" + shufflePort + + ", waitQueueSize= " + waitQueueSize + + ", enablePreemption= " + enablePreemption; + LOG.warn(logMsg); + final String currTSISO8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(new Date()); + // Time based log retrieval may not fetch the above log line so logging to stderr for debugging purpose. + System.err.println(currTSISO8601 + " " + logMsg); + long memRequired = executorMemoryBytes + (ioEnabled && isDirectCache == false ? ioMemoryBytes : 0); diff --git a/llap-server/src/main/resources/llap-daemon-log4j2.properties b/llap-server/src/main/resources/llap-daemon-log4j2.properties index 1c797dc..c901727 100644 --- a/llap-server/src/main/resources/llap-daemon-log4j2.properties +++ b/llap-server/src/main/resources/llap-daemon-log4j2.properties @@ -64,7 +64,7 @@ appender.RFA.strategy.max = ${sys:llap.daemon.log.maxbackupindex} appender.HISTORYAPPENDER.type = RollingRandomAccessFile appender.HISTORYAPPENDER.name = HISTORYAPPENDER appender.HISTORYAPPENDER.fileName = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.historylog.file} -appender.HISTORYAPPENDER.filePattern = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.historylog.file}_%d{yyyy-MM-dd}_%i.done +appender.HISTORYAPPENDER.filePattern = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.historylog.file}_%d{yyyy-MM-dd-HH}_%i.done appender.HISTORYAPPENDER.layout.type = PatternLayout appender.HISTORYAPPENDER.layout.pattern = %m%n appender.HISTORYAPPENDER.policies.type = Policies