diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java index 1f3b930..8cd6df7 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java @@ -399,6 +399,14 @@ private void run(String[] args) throws Exception { IOUtils.copyBytes(loggerContent, lfs.create(new Path(confPath, "llap-daemon-log4j2.properties"), true), conf, true); + URL metrics2 = conf.getResource(LlapDaemon.HADOOP_METRICS2_PROPERTIES_FILE); + if (metrics2 != null) { + InputStream metrics2FileStream = metrics2.openStream(); + IOUtils.copyBytes(metrics2FileStream, + lfs.create(new Path(confPath, LlapDaemon.HADOOP_METRICS2_PROPERTIES_FILE), true), + conf, true); + } + PrintWriter udfStream = new PrintWriter(lfs.create(new Path(confPath, StaticPermanentFunctionChecker.PERMANENT_FUNCTIONS_LIST))); for (String udfClass : allowedUdfs) { diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index c8734a5..2bc2ce7 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -73,6 +73,7 @@ private static final Logger LOG = LoggerFactory.getLogger(LlapDaemon.class); public static final String LOG4j2_PROPERTIES_FILE = "llap-daemon-log4j2.properties"; + public static final String HADOOP_METRICS2_PROPERTIES_FILE = "hadoop-metrics2.properties"; private final Configuration shuffleHandlerConf; private final LlapProtocolServerImpl server; private final ContainerRunnerImpl containerRunner; diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java index 52057e4..b89c6c4 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java @@ -45,7 +45,7 @@ /** * Llap daemon cache metrics source. */ -@Metrics(about = "LlapDaemon Cache Metrics", context = MetricsUtils.METRICS_CONTEXT) +@Metrics(about = "LlapDaemon Cache Metrics", context = "cache") public class LlapDaemonCacheMetrics implements MetricsSource { final String name; private String sessionId; @@ -127,7 +127,7 @@ public long getCacheHitBytes() { @Override public void getMetrics(MetricsCollector collector, boolean b) { MetricsRecordBuilder rb = collector.addRecord(CacheMetrics) - .setContext(MetricsUtils.METRICS_CONTEXT) + .setContext("cache") .tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME) .tag(SessionId, sessionId); getCacheStats(rb); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java index 33b8f9d..7919200 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java @@ -52,7 +52,7 @@ /** * Metrics about the llap daemon executors. */ -@Metrics(about = "LlapDaemon Executor Metrics", context = MetricsUtils.METRICS_CONTEXT) +@Metrics(about = "LlapDaemon Executor Metrics", context = "executors") public class LlapDaemonExecutorMetrics implements MetricsSource { private final String name; @@ -117,7 +117,7 @@ public static LlapDaemonExecutorMetrics create(String displayName, String sessio @Override public void getMetrics(MetricsCollector collector, boolean b) { MetricsRecordBuilder rb = collector.addRecord(ExecutorMetrics) - .setContext(MetricsUtils.METRICS_CONTEXT) + .setContext("executors") .tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME) .tag(SessionId, sessionId); getExecutorStats(rb); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonQueueMetrics.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonQueueMetrics.java index 5905cae..10a0124 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonQueueMetrics.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonQueueMetrics.java @@ -38,7 +38,7 @@ /** * */ -@Metrics(about = "LlapDaemon Queue Metrics", context = MetricsUtils.METRICS_CONTEXT) +@Metrics(about = "LlapDaemon Queue Metrics", context = "queue") public class LlapDaemonQueueMetrics implements MetricsSource { private final String name; private final String sessionId; @@ -78,7 +78,7 @@ public static LlapDaemonQueueMetrics create(String displayName, String sessionId @Override public void getMetrics(MetricsCollector collector, boolean b) { MetricsRecordBuilder rb = collector.addRecord(QueueMetrics) - .setContext(MetricsUtils.METRICS_CONTEXT) + .setContext("queue") .tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME) .tag(SessionId, sessionId); getQueueStats(rb); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/MetricsUtils.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/MetricsUtils.java index ce1c965..c22ce4e 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/MetricsUtils.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/MetricsUtils.java @@ -26,7 +26,6 @@ */ public class MetricsUtils { private static final String LOCALHOST = "localhost"; - public static final String METRICS_CONTEXT = "llap"; public static final String METRICS_PROCESS_NAME = "LlapDaemon"; diff --git a/llap-server/src/main/resources/hadoop-metrics2.properties.template b/llap-server/src/main/resources/hadoop-metrics2.properties.template new file mode 100644 index 0000000..994acaa --- /dev/null +++ b/llap-server/src/main/resources/hadoop-metrics2.properties.template @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# syntax: [prefix].[source|sink].[instance].[options] +# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details + +#*.sink.file.class=org.apache.hadoop.metrics2.sink.FileSink +# default sampling period, in seconds +#*.sink.file.period=10 + +# *.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +# *.sink.timeline.period=60 + +# llapdeamon metrics for all contexts (jvm,queue,executors,cache) will go to this file +# llapdaemon.sink.file.filename=llapdaemon-metrics.out + +# to configure separate files per context define following for each context +# llapdaemon.sink.file_jvm.class=org.apache.hadoop.metrics2.sink.FileSink +# llapdaemon.sink.file_jvm.context=jvm +# llapdaemon.sink.file_jvm.filename=llapdaemon-jvm-metrics.out