diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ebbc12d..a2a2529 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1385,6 +1385,13 @@ private static void addDeprecatedKeys() { public static final long DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS = 1000; + /** Timeline client policy for whether connections are fatal */ + public static final String TIMELINE_SERVICE_CLIENT_BEST_EFFORT = + TIMELINE_SERVICE_CLIENT_PREFIX + "best-effort"; + + public static final boolean + DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT = false; + /** Flag to enable recovery of timeline service */ public static final String TIMELINE_SERVICE_RECOVERY_ENABLED = TIMELINE_SERVICE_PREFIX + "recovery.enabled"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 975c267..a6a408f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -129,6 +129,7 @@ @VisibleForTesting String timelineDTRenewer; protected boolean timelineServiceEnabled; + protected boolean timelineServiceBestEffort; private static final String ROOT = "root"; @@ -163,14 +164,22 @@ protected void serviceInit(Configuration conf) throws Exception { if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { timelineServiceEnabled = true; - timelineClient = TimelineClient.createTimelineClient(); + timelineClient = createTimelineClient(); timelineClient.init(conf); timelineDTRenewer = getTimelineDelegationTokenRenewer(conf); timelineService = TimelineUtils.buildTimelineTokenService(conf); } + + timelineServiceBestEffort = conf.getBoolean( + YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT); super.serviceInit(conf); } + TimelineClient createTimelineClient() throws IOException, YarnException { + return TimelineClient.createTimelineClient(); + } + @Override protected void serviceStart() throws Exception { try { @@ -325,7 +334,15 @@ private void addTimelineDelegationToken( @VisibleForTesting org.apache.hadoop.security.token.Token getTimelineDelegationToken() throws IOException, YarnException { - return timelineClient.getDelegationToken(timelineDTRenewer); + try { + return timelineClient.getDelegationToken(timelineDTRenewer); + } catch (Exception e ) { + if (timelineServiceBestEffort) { + LOG.warn("Failed to get delegation token from the timeline server"); + return null; + } + throw e; + } } private static String getTimelineDelegationTokenRenewer(Configuration conf) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index eb8ca48..b3f632c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -854,6 +854,42 @@ private void testAsyncAPIPollTimeoutHelper(Long valueForTimeout, } @Test + public void testBestEffortTimelineDelegationToken() + throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf); + + YarnClientImpl client = spy(new YarnClientImpl() { + + @Override + TimelineClient createTimelineClient() throws IOException, YarnException { + timelineClient = mock(TimelineClient.class); + when(timelineClient.getDelegationToken(any(String.class))) + .thenThrow(new IOException()); + return timelineClient; + } + }); + + client.init(conf); + try { + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, true); + client.serviceInit(conf); + client.getTimelineDelegationToken(); + } catch (Exception e) { + Assert.fail("Should not have thrown an exception"); + } + try { + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, false); + client.serviceInit(conf); + client.getTimelineDelegationToken(); + Assert.fail("Get delegation token should have thrown an exception"); + } catch (Exception e) { + // Success + } + } + + @Test public void testAutomaticTimelineDelegationTokenLoading() throws Exception { Configuration conf = new YarnConfiguration(); @@ -864,22 +900,18 @@ public void testAutomaticTimelineDelegationTokenLoading() final Token dToken = new Token( timelineDT.getBytes(), new byte[0], timelineDT.getKind(), new Text()); - // crate a mock client + // create a mock client YarnClientImpl client = spy(new YarnClientImpl() { + @Override - protected void serviceInit(Configuration conf) throws Exception { - if (getConfig().getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { - timelineServiceEnabled = true; - timelineClient = mock(TimelineClient.class); - when(timelineClient.getDelegationToken(any(String.class))) - .thenReturn(dToken); - timelineClient.init(getConfig()); - timelineService = TimelineUtils.buildTimelineTokenService(getConfig()); - } - this.setConfig(conf); + TimelineClient createTimelineClient() throws IOException, YarnException { + timelineClient = mock(TimelineClient.class); + when(timelineClient.getDelegationToken(any(String.class))) + .thenReturn(dToken); + return timelineClient; } + @Override protected void serviceStart() throws Exception { rmClient = mock(ApplicationClientProtocol.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index f1e6e67..f662ee1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1370,6 +1370,12 @@ + Client policy for whether timeline operations are non-fatal + yarn.timeline-service.client.best-effort + false + + + Default retry time interval for timeline servive client.