diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 93f7ed6..a0bef13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1913,6 +1913,11 @@ private static void addDeprecatedKeys() { public static final String YARN_HTTP_POLICY_KEY = YARN_PREFIX + "http.policy"; public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY .name(); + + /** + * Tracing configurations + */ + public static final String YARN_SERVER_HTRACE_PREFIX = YARN_PREFIX + "htrace."; /** * Node-labels configurations diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestYARNTracing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestYARNTracing.java new file mode 100644 index 0000000..7bfa1f5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestYARNTracing.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.tracing.SetSpanReceiver; +import org.apache.hadoop.tracing.SpanReceiverHost; +import org.apache.hadoop.tracing.TraceAdmin; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.MiniYARNCluster; +import org.apache.htrace.Sampler; +import org.apache.htrace.Trace; +import org.apache.htrace.TraceScope; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestYARNTracing { + private static MiniYARNCluster cluster; + + @BeforeClass + public static void setup() throws IOException { + Configuration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.YARN_SERVER_HTRACE_PREFIX + + SpanReceiverHost.SPAN_RECEIVERS_CONF_SUFFIX, + SetSpanReceiver.class.getName()); + cluster = new MiniYARNCluster(TestYARNTracing.class.getSimpleName(), + 1, 1, 1); + cluster.init(conf); + cluster.start(); + } + + @AfterClass + public static void teardown() { + if (cluster != null) { + cluster.stop(); + cluster = null; + } + } + + @Before + public void clearSpans() { + SetSpanReceiver.clear(); + } + + @Test + public void testRMTracing() throws Exception { + YarnClient client = YarnClient.createYarnClient(); + client.init(cluster.getConfig()); + client.start(); + + try (TraceScope ts = Trace.startSpan("testRMTracing", Sampler.ALWAYS)) { + client.getApplications(); + } + + client.stop(); + String[] expectedSpanNames = { + "testRMTracing", + "ApplicationClientProtocolPB#getApplications", + "ApplicationClientProtocolService#getApplications" + }; + SetSpanReceiver.assertSpanNamesFound(expectedSpanNames); + } + + @Test + public void testRMTraceAdmin() throws Exception { + Configuration conf = cluster.getConfig(); + String hostPort = conf.get(YarnConfiguration.RM_ADMIN_ADDRESS, + YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS); + TraceAdmin traceAdmin = new TraceAdmin(); + traceAdmin.setConf(conf); + + Assert.assertEquals(0, + runTraceCommand(traceAdmin, "-list", "-host", hostPort)); + Assert.assertEquals(0, + runTraceCommand(traceAdmin, "-remove", "1","-host", hostPort)); + Assert.assertEquals(0, + runTraceCommand(traceAdmin, + "-add", "-class", SetSpanReceiver.class.getName(), + "-host", hostPort)); + + try (TraceScope ts = Trace.startSpan("traceAdmin", Sampler.ALWAYS)) { + runTraceCommand(traceAdmin, "-list", "-host", hostPort); + } + + String[] expectedSpanNames = { + "traceAdmin", + "TraceAdminService#listSpanReceivers", + }; + SetSpanReceiver.assertSpanNamesFound(expectedSpanNames); + } + + private static int runTraceCommand(TraceAdmin trace, String... cmd) + throws Exception { + return trace.run(cmd); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index e5bb6e5..cfd1ef2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -40,6 +40,11 @@ import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RPC.Server; import org.apache.hadoop.ipc.StandbyException; +import org.apache.hadoop.tracing.SpanReceiverInfo; +import org.apache.hadoop.tracing.TraceAdminProtocol; +import org.apache.hadoop.tracing.TraceAdminProtocolPB; +import org.apache.hadoop.tracing.TraceAdminProtocolServerSideTranslatorPB; +import org.apache.hadoop.tracing.TraceAdminPB.TraceAdminService; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.UserGroupInformation; @@ -91,7 +96,9 @@ import com.google.protobuf.BlockingService; public class AdminService extends CompositeService implements - HAServiceProtocol, ResourceManagerAdministrationProtocol { + HAServiceProtocol, + ResourceManagerAdministrationProtocol, + TraceAdminProtocol { private static final Log LOG = LogFactory.getLog(AdminService.class); @@ -204,6 +211,15 @@ protected void startServer() throws Exception { HAServiceProtocol.class, haPbService); } + TraceAdminProtocolServerSideTranslatorPB traceAdminXlator = + new TraceAdminProtocolServerSideTranslatorPB(this); + RPC.setProtocolEngine(conf, TraceAdminProtocolPB.class, + ProtobufRpcEngine.class); + BlockingService traceAdminService = + TraceAdminService.newReflectiveBlockingService(traceAdminXlator); + server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, + TraceAdminProtocolPB.class, traceAdminService); + this.server.start(); conf.updateConnectAddr(YarnConfiguration.RM_BIND_HOST, YarnConfiguration.RM_ADMIN_ADDRESS, @@ -742,4 +758,22 @@ public CheckForDecommissioningNodesResponse checkForDecommissioningNodes( response.setDecommissioningNodes(decommissioningNodes); return response; } + + @Override // TraceAdminProtocol + public SpanReceiverInfo[] listSpanReceivers() throws IOException { + checkAccess("listSpanReceivers"); + return rm.listSpanReceivers(); + } + + @Override // TraceAdminProtocol + public long addSpanReceiver(SpanReceiverInfo info) throws IOException { + checkAccess("addSpanReceiver"); + return rm.addSpanReceiver(info); + } + + @Override // TraceAdminProtocol + public void removeSpanReceiver(long id) throws IOException { + checkAccess("removeSpanReceiver"); + rm.removeSpanReceiver(id); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index d6d9629..0c8da33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -37,6 +37,8 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.service.Service; +import org.apache.hadoop.tracing.SpanReceiverHost; +import org.apache.hadoop.tracing.SpanReceiverInfo; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.JvmPauseMonitor; @@ -159,6 +161,7 @@ private AppReportFetcher fetcher = null; protected ResourceTrackerService resourceTracker; private JvmPauseMonitor pauseMonitor; + private SpanReceiverHost spanReceiverHost; @VisibleForTesting protected String webAppAddress; @@ -259,6 +262,9 @@ protected void serviceInit(Configuration conf) throws Exception { YarnConfiguration.RM_BIND_HOST, WebAppUtils.getRMWebAppURLWithoutScheme(this.conf)); + this.spanReceiverHost = + SpanReceiverHost.get(conf, YarnConfiguration.YARN_SERVER_HTRACE_PREFIX); + RMApplicationHistoryWriter rmApplicationHistoryWriter = createRMApplicationHistoryWriter(); addService(rmApplicationHistoryWriter); @@ -1084,6 +1090,9 @@ protected void serviceStop() throws Exception { super.serviceStop(); transitionToStandby(false); rmContext.setHAServiceState(HAServiceState.STOPPING); + if (spanReceiverHost != null) { + spanReceiverHost.closeReceivers(); + } } protected ResourceTrackerService createResourceTrackerService() { @@ -1277,6 +1286,18 @@ private static void removeApplication(Configuration conf, String applicationId) } } + SpanReceiverInfo[] listSpanReceivers() throws IOException { + return spanReceiverHost.listSpanReceivers(); + } + + long addSpanReceiver(SpanReceiverInfo info) throws IOException { + return spanReceiverHost.addSpanReceiver(info); + } + + void removeSpanReceiver(long id) throws IOException { + spanReceiverHost.removeSpanReceiver(id); + } + private static void printUsage(PrintStream out) { out.println("Usage: yarn resourcemanager [-format-state-store]"); out.println(" "