diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 54e8888f0d2..a36f4d156be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -38,6 +38,7 @@ import org.apache.hadoop.ha.ActiveStandbyElector; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.util.BasicDiskValidator; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; @@ -3257,6 +3258,18 @@ public static boolean isAclEnabled(Configuration conf) { public static final long DEFAULT_TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS = 2000L; + /** + * The configuration prefix of timeline HTTP authentication. + */ + public static final String TIMELINE_HTTP_AUTH_PREFIX = + TIMELINE_SERVICE_PREFIX + "http-authentication."; + + /** + * The authentication type for timeline HTTP authentication. + */ + public static final String TIMELINE_HTTP_AUTH_TYPE = + TIMELINE_HTTP_AUTH_PREFIX + AuthenticationFilter.AUTH_TYPE; + // mark app-history related configs @Private as application history is going // to be integrated into the timeline service @Private diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 14133ba4ecd..4f24a3075b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -40,6 +40,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; @@ -310,7 +311,9 @@ public YarnClientApplication createApplication() // Automatically add the timeline DT into the CLC // Only when the security and the timeline service are both enabled - if (isSecurityEnabled() && timelineV1ServiceEnabled) { + if (isSecurityEnabled() && timelineV1ServiceEnabled && + getConfig().get(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE) + .equals(KerberosAuthenticationHandler.TYPE)) { addTimelineDelegationToken(appContext.getAMContainerSpec()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 4c9d46e09e4..08437c0a2c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; @@ -1228,6 +1229,9 @@ private void testCreateTimelineClientWithError( timelineClientBestEffort); conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, timelineVersion); + // Timeline Delegation token and client is only used for kerberos + conf.set(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE, + KerberosAuthenticationHandler.TYPE); MockYarnClient client = new MockYarnClient(); MockYarnClient spyClient = spy(client); when(spyClient.createTimelineClient()).thenThrow(mockErr); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientImpl.java index a6259a7be05..f9597a9b1be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientImpl.java @@ -27,6 +27,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; @@ -117,7 +118,8 @@ public void testBestEffortTimelineDelegationToken() Configuration conf = getConf(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); SecurityUtil.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS, conf); - + conf.set(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE, + KerberosAuthenticationHandler.TYPE); YarnClientImpl client = spy(new YarnClientImpl() { @Override @@ -151,6 +153,8 @@ public void testAutomaticTimelineDelegationTokenLoading() Configuration conf = getConf(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); SecurityUtil.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS, conf); + conf.set(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE, + KerberosAuthenticationHandler.TYPE); TimelineDelegationTokenIdentifier timelineDT = new TimelineDelegationTokenIdentifier(); final Token dToken = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index 7eb4ec129c7..f5ca2a354c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -28,6 +28,7 @@ import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -349,7 +350,9 @@ private static void putTimelineDataInJSONFile(String path, String type) { client.start(); try { if (UserGroupInformation.isSecurityEnabled() - && conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) { + && conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false) + && conf.get(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE) + .equals(KerberosAuthenticationHandler.TYPE)) { Token token = client.getDelegationToken( UserGroupInformation.getCurrentUser().getUserName()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java index 2e878619614..75886a06e67 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java @@ -34,6 +34,8 @@ import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLSocketFactory; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -111,8 +113,12 @@ protected void serviceInit(Configuration conf) throws Exception { } else { connConfigurator = DEFAULT_TIMEOUT_CONN_CONFIGURATOR; } - - if (UserGroupInformation.isSecurityEnabled()) { + String defaultAuth = UserGroupInformation.isSecurityEnabled() ? + KerberosAuthenticationHandler.TYPE : + PseudoAuthenticationHandler.TYPE; + String authType = conf.get(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE, + defaultAuth); + if (authType.equals(KerberosAuthenticationHandler.TYPE)) { authenticator = new KerberosDelegationTokenAuthenticator(); } else { authenticator = new PseudoDelegationTokenAuthenticator(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java index d95ee7723b9..9f9564a3106 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java @@ -243,6 +243,7 @@ public void testDelegationTokenOperationsRetry() throws Exception { // use kerberos to bypass the issue in HADOOP-11215 conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + conf.set(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE, "kerberos"); UserGroupInformation.setConfiguration(conf); TimelineClientImpl client = createTimelineClient(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java index d918e8ddde6..0e1310eb696 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java @@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer; import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineStore; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -107,11 +108,11 @@ public static void setup() { try { testTimelineServer = new ApplicationHistoryServer(); conf = new Configuration(false); - conf.setStrings(TimelineAuthenticationFilterInitializer.PREFIX + "type", + conf.setStrings(TIMELINE_HTTP_AUTH_PREFIX + "type", "kerberos"); - conf.set(TimelineAuthenticationFilterInitializer.PREFIX + + conf.set(TIMELINE_HTTP_AUTH_PREFIX + KerberosAuthenticationHandler.PRINCIPAL, httpSpnegoPrincipal); - conf.set(TimelineAuthenticationFilterInitializer.PREFIX + + conf.set(TIMELINE_HTTP_AUTH_PREFIX + KerberosAuthenticationHandler.KEYTAB, httpSpnegoKeytabFile.getAbsolutePath()); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java index 96c3cdf420d..944c076b881 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java @@ -31,6 +31,7 @@ import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler; import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler; import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX; import java.util.HashMap; import java.util.Map; @@ -48,12 +49,6 @@ */ public class TimelineAuthenticationFilterInitializer extends FilterInitializer { - /** - * The configuration prefix of timeline HTTP authentication. - */ - public static final String PREFIX = - "yarn.timeline-service.http-authentication."; - @VisibleForTesting Map filterConfig; @@ -68,7 +63,8 @@ protected void setAuthFilterConfig(Configuration conf) { // yarn.timeline-service.http-authentication.proxyuser will override // hadoop.proxyuser Map timelineAuthProps = - AuthenticationFilterInitializer.getFilterConfigMap(conf, PREFIX); + AuthenticationFilterInitializer.getFilterConfigMap(conf, + TIMELINE_HTTP_AUTH_PREFIX); filterConfig.putAll(timelineAuthProps); } @@ -81,7 +77,8 @@ protected void setAuthFilterConfig(Configuration conf) { * Initializes {@link TimelineAuthenticationFilter}. *

* Propagates to {@link TimelineAuthenticationFilter} configuration all YARN - * configuration properties prefixed with {@value #PREFIX}. + * configuration properties prefixed with + * {@value org.apache.hadoop.yarn.conf.YarnConfiguration#TIMELINE_HTTP_AUTH_PREFIX}. * * @param container * The filter container. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java index 44f63ead34a..25996709aa6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java @@ -23,7 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.FilterContainer; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import static org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer.PREFIX; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX; import org.junit.Test; import org.mockito.Mockito; @@ -46,9 +46,9 @@ public void testProxyUserConfiguration() { break; case 1: // yarn.timeline-service.http-authentication.proxyuser prefix - conf.set(PREFIX + "proxyuser.foo.hosts", "*"); - conf.set(PREFIX + "proxyuser.foo.users", "*"); - conf.set(PREFIX + "proxyuser.foo.groups", "*"); + conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.hosts", "*"); + conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.users", "*"); + conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.groups", "*"); break; case 2: // hadoop.proxyuser prefix has been overwritten by @@ -56,9 +56,9 @@ public void testProxyUserConfiguration() { conf.set("hadoop.proxyuser.foo.hosts", "bar"); conf.set("hadoop.proxyuser.foo.users", "bar"); conf.set("hadoop.proxyuser.foo.groups", "bar"); - conf.set(PREFIX + "proxyuser.foo.hosts", "*"); - conf.set(PREFIX + "proxyuser.foo.users", "*"); - conf.set(PREFIX + "proxyuser.foo.groups", "*"); + conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.hosts", "*"); + conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.users", "*"); + conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.groups", "*"); break; default: break; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java index 0c70a5afdab..f773807f05d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java @@ -68,13 +68,13 @@ import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol; import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse; -import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector; import org.apache.hadoop.yarn.server.timelineservice.collector.NodeTimelineCollectorManager; import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService; import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -151,11 +151,11 @@ public static void setup() { conf = new Configuration(false); conf.setClass("fs.file.impl", RawLocalFileSystem.class, FileSystem.class); - conf.setStrings(TimelineAuthenticationFilterInitializer.PREFIX + "type", + conf.setStrings(TIMELINE_HTTP_AUTH_PREFIX + "type", "kerberos"); - conf.set(TimelineAuthenticationFilterInitializer.PREFIX + + conf.set(TIMELINE_HTTP_AUTH_PREFIX + KerberosAuthenticationHandler.PRINCIPAL, httpSpnegoPrincipal); - conf.set(TimelineAuthenticationFilterInitializer.PREFIX + + conf.set(TIMELINE_HTTP_AUTH_PREFIX + KerberosAuthenticationHandler.KEYTAB, httpSpnegoKeytabFile.getAbsolutePath()); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,