diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDelegationTokenResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDelegationTokenResponse.java new file mode 100644 index 0000000..90d13a2 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDelegationTokenResponse.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.timeline; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * The response of delegation token related request + */ +@XmlRootElement(name = "delegationtoken") +@XmlAccessorType(XmlAccessType.NONE) +@Public +@Unstable +public class TimelineDelegationTokenResponse { + + private String type; + private Object content; + + public TimelineDelegationTokenResponse() { + + } + + @XmlElement(name = "type") + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + @XmlElement(name = "content") + public Object getContent() { + return content; + } + + public void setContent(Object content) { + this.content = content; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index fe3c1e1..f67767e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1144,6 +1144,10 @@ public static final String TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = TIMELINE_SERVICE_PREFIX + "webapp.spnego-keytab-file"; + /**The signature secret file to be used for spnego filter for timeline service.*/ + public static final String TIMELINE_SERVICE_WEBAPP_SIGNATURE_SECRET_FILE_KEY = + TIMELINE_SERVICE_PREFIX + "webapp.signature-secret-file"; + /** Timeline service store class */ public static final String TIMELINE_SERVICE_STORE = TIMELINE_SERVICE_PREFIX + "store-class"; @@ -1196,6 +1200,14 @@ public static final long DEFAULT_TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS = 1000 * 60 * 5; + /** The keytab for the timeline server.*/ + public static final String TIMELINE_SERVICE_KEYTAB = + TIMELINE_SERVICE_PREFIX + "keytab"; + + /** The Kerberos principal for the timeline server.*/ + public static final String TIMELINE_SERVICE_PRINCIPAL = + TIMELINE_SERVICE_PREFIX + "principal"; + //////////////////////////////// // Other Configs //////////////////////////////// diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTimelineSecurityInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTimelineSecurityInfo.java new file mode 100644 index 0000000..d2b1358 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTimelineSecurityInfo.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.security.client; + +import java.lang.annotation.Annotation; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.KerberosInfo; +import org.apache.hadoop.security.SecurityInfo; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.TokenInfo; +import org.apache.hadoop.security.token.TokenSelector; +import org.apache.hadoop.yarn.api.ApplicationHistoryProtocolPB; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + + +@Public +@Unstable +public class ClientTimelineSecurityInfo extends SecurityInfo { + @Override + public KerberosInfo getKerberosInfo(Class protocol, Configuration conf) { + if (!protocol + .equals(ApplicationHistoryProtocolPB.class)) { + return null; + } + return new KerberosInfo() { + + @Override + public Class annotationType() { + return null; + } + + @Override + public String serverPrincipal() { + return YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL; + } + + @Override + public String clientPrincipal() { + return null; + } + }; + } + + @Override + public TokenInfo getTokenInfo(Class protocol, Configuration conf) { + if (!protocol + .equals(ApplicationHistoryProtocolPB.class)) { + return null; + } + return new TokenInfo() { + + @Override + public Class annotationType() { + return null; + } + + @Override + public Class> + value() { + return TimelineDelegationTokenSelector.class; + } + }; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenIdentifier.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenIdentifier.java new file mode 100644 index 0000000..82e0d69 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenIdentifier.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.security.client; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; + +@Public +@Unstable +public class TimelineDelegationTokenIdentifier extends AbstractDelegationTokenIdentifier { + + public static final Text KIND_NAME = new Text("TIMELINE_DELEGATION_TOKEN"); + + public TimelineDelegationTokenIdentifier() { + + } + + /** + * Create a new timeline delegation token identifier + * + * @param owner the effective username of the token owner + * @param renewer the username of the renewer + * @param realUser the real username of the token owner + */ + public TimelineDelegationTokenIdentifier(Text owner, Text renewer, + Text realUser) { + super(owner, renewer, realUser); + } + + @Override + public Text getKind() { + return KIND_NAME; + } + + @InterfaceAudience.Private + public static class Renewer extends Token.TrivialRenewer { + @Override + protected Text getKind() { + return KIND_NAME; + } + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenSelector.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenSelector.java new file mode 100644 index 0000000..62f708a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenSelector.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.security.client; + +import java.util.Collection; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.TokenSelector; + +@Public +@Unstable +public class TimelineDelegationTokenSelector + implements TokenSelector { + + private static final Log LOG = LogFactory + .getLog(TimelineDelegationTokenSelector.class); + + @SuppressWarnings("unchecked") + public Token selectToken(Text service, + Collection> tokens) { + if (service == null) { + return null; + } + LOG.debug("Looking for a token with service " + service.toString()); + for (Token token : tokens) { + LOG.debug("Token kind is " + token.getKind().toString() + + " and the token's service name is " + token.getService()); + if (TimelineDelegationTokenIdentifier.KIND_NAME.equals(token.getKind()) + && service.equals(token.getService())) { + return (Token) token; + } + } + return null; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineKerberosAuthenticator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineKerberosAuthenticator.java new file mode 100644 index 0000000..fb1cb8e --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineKerberosAuthenticator.java @@ -0,0 +1,282 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.security.client; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.client.Authenticator; +import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPut; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; + +/** + * A KerberosAuthenticator subclass that fallback to + * {@link TimelineKerberosAuthenticator}. + */ + +@Private +@Unstable +public class TimelineKerberosAuthenticator extends KerberosAuthenticator { + + public static final String ERROR_EXCEPTION_JSON = "exception"; + public static final String ERROR_CLASSNAME_JSON = "javaClassName"; + public static final String ERROR_MESSAGE_JSON = "message"; + + public static final String OP_PARAM = "op"; + public static final String DELEGATION_PARAM = "delegation"; + public static final String TOKEN_PARAM = "token"; + public static final String RENEWER_PARAM = "renewer"; + public static final String DELEGATION_TOKEN_URL = "url"; + public static final String DELEGATION_TOKEN_EXPIRATION_TIME = + "expirationTime"; + + private static ObjectMapper mapper; + + static { + mapper = new ObjectMapper(); + YarnJacksonJaxbJsonProvider.configObjectMapper(mapper); + } + + /** + * Returns the fallback authenticator if the server does not use Kerberos + * SPNEGO HTTP authentication. + * + * @return a {@link TimelineKerberosAuthenticator} instance. + */ + @Override + protected Authenticator getFallBackAuthenticator() { + return new TimelineKerberosAuthenticator(); + } + + /** + * DelegationToken operations. + */ + @Private + public static enum DelegationTokenOperation { + GETDELEGATIONTOKEN(HttpGet.METHOD_NAME, true), + RENEWDELEGATIONTOKEN(HttpPut.METHOD_NAME, true), + CANCELDELEGATIONTOKEN(HttpPut.METHOD_NAME, false); + + private String httpMethod; + private boolean requiresKerberosCredentials; + + private DelegationTokenOperation(String httpMethod, + boolean requiresKerberosCredentials) { + this.httpMethod = httpMethod; + this.requiresKerberosCredentials = requiresKerberosCredentials; + } + + public String getHttpMethod() { + return httpMethod; + } + + public boolean requiresKerberosCredentials() { + return requiresKerberosCredentials; + } + + } + + public static void injectDelegationToken(Map params, + Token dtToken) + throws IOException { + if (dtToken != null) { + params.put(DELEGATION_PARAM, dtToken.encodeToUrlString()); + } + } + + private boolean hasDelegationToken(URL url) { + return url.getQuery().contains(DELEGATION_PARAM + "="); + } + + @Override + public void authenticate(URL url, AuthenticatedURL.Token token) + throws IOException, AuthenticationException { + if (!hasDelegationToken(url)) { + super.authenticate(url, token); + } + } + + public static Token getDelegationToken( + URL url, AuthenticatedURL.Token token, String renewer) throws IOException { + DelegationTokenOperation op = + DelegationTokenOperation.GETDELEGATIONTOKEN; + Map params = new HashMap(); + params.put(OP_PARAM, op.toString()); + params.put(RENEWER_PARAM, renewer); + url = appendParams(url, params); + AuthenticatedURL aUrl = + new AuthenticatedURL(new TimelineKerberosAuthenticator()); + try { + HttpURLConnection conn = aUrl.openConnection(url, token); + conn.setRequestMethod(op.getHttpMethod()); + TimelineDelegationTokenResponse dtRes = + validateAndParseResponse(conn, HttpURLConnection.HTTP_OK); + if (!dtRes.getType().equals(DELEGATION_TOKEN_URL)) { + throw new IOException("The response content is not expected: " + + dtRes.getContent()); + } + String tokenStr = dtRes.getContent().toString(); + Token dToken = + new Token(); + dToken.decodeFromUrlString(tokenStr); + return dToken; + } catch (AuthenticationException ex) { + throw new IOException(ex.toString(), ex); + } + } + + public static long renewDelegationToken(URL url, + AuthenticatedURL.Token token, + Token dToken) throws IOException { + Map params = new HashMap(); + params.put(OP_PARAM, + DelegationTokenOperation.RENEWDELEGATIONTOKEN.toString()); + params.put(TOKEN_PARAM, dToken.encodeToUrlString()); + url = appendParams(url, params); + AuthenticatedURL aUrl = + new AuthenticatedURL(new TimelineKerberosAuthenticator()); + try { + HttpURLConnection conn = aUrl.openConnection(url, token); + conn.setRequestMethod( + DelegationTokenOperation.RENEWDELEGATIONTOKEN.getHttpMethod()); + TimelineDelegationTokenResponse dtRes = + validateAndParseResponse(conn, HttpURLConnection.HTTP_OK); + if (!dtRes.getType().equals(DELEGATION_TOKEN_EXPIRATION_TIME)) { + throw new IOException("The response content is not expected: " + + dtRes.getContent()); + } + return Long.valueOf(dtRes.getContent().toString()); + } catch (AuthenticationException ex) { + throw new IOException(ex.toString(), ex); + } + } + + public static void cancelDelegationToken(URL url, + AuthenticatedURL.Token token, + Token dToken) throws IOException { + Map params = new HashMap(); + params.put(OP_PARAM, + DelegationTokenOperation.CANCELDELEGATIONTOKEN.toString()); + params.put(TOKEN_PARAM, dToken.encodeToUrlString()); + url = appendParams(url, params); + AuthenticatedURL aUrl = + new AuthenticatedURL(new TimelineKerberosAuthenticator()); + try { + HttpURLConnection conn = aUrl.openConnection(url, token); + conn.setRequestMethod(DelegationTokenOperation.CANCELDELEGATIONTOKEN + .getHttpMethod()); + validateAndParseResponse(conn, HttpURLConnection.HTTP_OK); + } catch (AuthenticationException ex) { + throw new IOException(ex.toString(), ex); + } + } + + /** + * Convenience method that appends parameters an HTTP URL. + * + * @param path + * the file path. + * @param params + * the query string parameters. + * + * @return a URL + * + * @throws IOException + * thrown if an IO error occurs. + */ + public static URL appendParams(URL url, Map params) + throws IOException { + StringBuilder sb = new StringBuilder(); + sb.append(url); + String separator = url.toString().contains("?") ? "&" : "?"; + for (Map.Entry entry : params.entrySet()) { + sb.append(separator).append(entry.getKey()).append("="). + append(URLEncoder.encode(entry.getValue(), "UTF8")); + separator = "&"; + } + return new URL(sb.toString()); + } + + /** + * Validates the status of an HttpURLConnection against an + * expected HTTP status code. If the current status code is not the expected + * one it throws an exception with a detail message using Server side error + * messages if available. Otherwise, {@link TimelineDelegationTokenResponse} + * will be parsed and returned. + * + * @param conn + * the HttpURLConnection. + * @param expected + * the expected HTTP status code. + * @return + * @throws IOException + * thrown if the current status code does not match the expected one + * or the JSON response cannot be parsed correctly + */ + private static TimelineDelegationTokenResponse validateAndParseResponse( + HttpURLConnection conn, int expected) throws IOException { + int status = conn.getResponseCode(); + JsonNode json = mapper.readTree(conn.getInputStream()); + if (status != expected) { + try { + String message = json.get(ERROR_MESSAGE_JSON).getTextValue(); + String exception = json.get(ERROR_EXCEPTION_JSON).getTextValue(); + String className = json.get(ERROR_CLASSNAME_JSON).getTextValue(); + + try { + ClassLoader cl = TimelineKerberosAuthenticator.class.getClassLoader(); + Class klass = cl.loadClass(className); + Constructor constr = klass.getConstructor(String.class); + throw (IOException) constr.newInstance(message); + } catch (IOException ex) { + throw ex; + } catch (Exception ex) { + throw new IOException(MessageFormat.format("{0} - {1}", exception, + message)); + } + } catch (IOException ex) { + if (ex.getCause() instanceof IOException) { + throw (IOException) ex.getCause(); + } + throw new IOException( + MessageFormat.format("HTTP status [{0}], {1}", + status, conn.getResponseMessage())); + } + } else { + return mapper.readValue(json, TimelineDelegationTokenResponse.class); + } + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java index 324c6f6..276db1a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.codehaus.jackson.JsonGenerationException; import org.codehaus.jackson.map.AnnotationIntrospector; import org.codehaus.jackson.map.JsonMappingException; @@ -41,9 +42,7 @@ static { mapper = new ObjectMapper(); - AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(); - mapper.setAnnotationIntrospector(introspector); - mapper.setSerializationInclusion(Inclusion.NON_NULL); + YarnJacksonJaxbJsonProvider.configObjectMapper(mapper); } /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java index f8c6f55..398a270 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java @@ -36,10 +36,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpConfig.Policy; -import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.AdminACLsManager; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.slf4j.Logger; @@ -92,6 +90,8 @@ boolean devMode = false; private String spnegoPrincipalKey; private String spnegoKeytabKey; + private String signatureSecretFileKey; + private String spnegoFilterClass; private final HashSet servlets = new HashSet(); private final HashMap attributes = new HashMap(); @@ -162,6 +162,17 @@ return this; } + public Builder withHttpSignatureSecretFileKey( + String signatureSecretFileKey) { + this.signatureSecretFileKey = signatureSecretFileKey; + return this; + } + + public Builder withHttpSpnegoFilterClass(String spnegoFilterClass) { + this.spnegoFilterClass = spnegoFilterClass; + return this; + } + public Builder inDevMode() { devMode = true; return this; @@ -250,6 +261,8 @@ public void setup() { if (hasSpnegoConf) { builder.setUsernameConfKey(spnegoPrincipalKey) .setKeytabConfKey(spnegoKeytabKey) + .setSignatureSecretFileKey(signatureSecretFileKey) + .setSpnegoFilterClass(spnegoFilterClass) .setSecurityEnabled(UserGroupInformation.isSecurityEnabled()); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java index ac0ea7f..3cc1aec 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java @@ -49,9 +49,14 @@ public YarnJacksonJaxbJsonProvider() { @Override public ObjectMapper locateMapper(Class type, MediaType mediaType) { ObjectMapper mapper = super.locateMapper(type, mediaType); + configObjectMapper(mapper); + return mapper; + } + + public static void configObjectMapper(ObjectMapper mapper) { AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(); mapper.setAnnotationIntrospector(introspector); mapper.setSerializationInclusion(Inclusion.NON_NULL); - return mapper; } + } \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo index babc2fb..d90a267 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo @@ -12,6 +12,7 @@ # limitations under the License. # org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo +org.apache.hadoop.yarn.security.client.ClientTimelineSecurityInfo org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo org.apache.hadoop.yarn.security.SchedulerSecurityInfo org.apache.hadoop.yarn.security.admin.AdminSecurityInfo diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier index d860461..a4ad548 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -15,4 +15,5 @@ org.apache.hadoop.yarn.security.ContainerTokenIdentifier org.apache.hadoop.yarn.security.AMRMTokenIdentifier org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier +org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier org.apache.hadoop.yarn.security.NMTokenIdentifier diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer index dd0b2c4..9fcfa19 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer @@ -14,3 +14,4 @@ org.apache.hadoop.yarn.security.AMRMTokenIdentifier$Renewer org.apache.hadoop.yarn.security.ContainerTokenIdentifier$Renewer org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier$Renewer +org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier$Renewer diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index 731ae14..047e881 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -18,12 +18,16 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; +import java.io.IOException; +import java.net.InetSocketAddress; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.service.Service; import org.apache.hadoop.util.ExitUtil; @@ -33,8 +37,10 @@ import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.security.TimelineAuthenticationFilter; +import org.apache.hadoop.yarn.server.applicationhistoryservice.security.TimelineDelegationTokenSecretManagerService; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; @@ -52,10 +58,11 @@ private static final Log LOG = LogFactory .getLog(ApplicationHistoryServer.class); - ApplicationHistoryClientService ahsClientService; - ApplicationHistoryManager historyManager; - TimelineStore timelineStore; - private WebApp webApp; + protected ApplicationHistoryClientService ahsClientService; + protected ApplicationHistoryManager historyManager; + protected TimelineStore timelineStore; + protected TimelineDelegationTokenSecretManagerService secretManagerService; + protected WebApp webApp; public ApplicationHistoryServer() { super(ApplicationHistoryServer.class.getName()); @@ -69,13 +76,21 @@ protected void serviceInit(Configuration conf) throws Exception { addService((Service) historyManager); timelineStore = createTimelineStore(conf); addIfService(timelineStore); + secretManagerService = createTimelineDelegationTokenSecretManagerService(conf); + addService(secretManagerService); + + DefaultMetricsSystem.initialize("ApplicationHistoryServer"); + JvmMetrics.initSingleton("ApplicationHistoryServer", null); super.serviceInit(conf); } @Override protected void serviceStart() throws Exception { - DefaultMetricsSystem.initialize("ApplicationHistoryServer"); - JvmMetrics.initSingleton("ApplicationHistoryServer", null); + try { + doSecureLogin(getConfig()); + } catch(IOException ie) { + throw new YarnRuntimeException("Failed to login", ie); + } startWebApp(); super.serviceStart(); @@ -148,27 +163,41 @@ protected TimelineStore createTimelineStore( TimelineStore.class), conf); } + protected TimelineDelegationTokenSecretManagerService + createTimelineDelegationTokenSecretManagerService(Configuration conf) { + return new TimelineDelegationTokenSecretManagerService(); + } + protected void startWebApp() { String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(getConfig()); LOG.info("Instantiating AHSWebApp at " + bindAddress); try { + AHSWebApp ahsWebApp = AHSWebApp.getInstance(); + ahsWebApp.setApplicationHistoryManager(historyManager); + ahsWebApp.setTimelineStore(timelineStore); + ahsWebApp.setTimelineDelegationTokenSecretManagerService(secretManagerService); webApp = WebApps .$for("applicationhistory", ApplicationHistoryClientService.class, - ahsClientService, "ws") + ahsClientService, "ws") .with(getConfig()) .withHttpSpnegoPrincipalKey( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_USER_NAME_KEY) + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_USER_NAME_KEY) .withHttpSpnegoKeytabKey( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) + .withHttpSignatureSecretFileKey( + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SIGNATURE_SECRET_FILE_KEY) + .withHttpSpnegoFilterClass( + TimelineAuthenticationFilter.class.getName()) .at(bindAddress) - .start(new AHSWebApp(historyManager, timelineStore)); + .start(ahsWebApp); } catch (Exception e) { String msg = "AHSWebApp failed to start."; LOG.error(msg, e); throw new YarnRuntimeException(msg, e); } } + /** * @return ApplicationTimelineStore */ @@ -177,4 +206,27 @@ protected void startWebApp() { public TimelineStore getTimelineStore() { return timelineStore; } + + protected TimelineDelegationTokenSecretManagerService + getTimelineDelegationTokenSecretManagerService() { + return secretManagerService; + } + + private void doSecureLogin(Configuration conf) throws IOException { + InetSocketAddress socAddr = getBindAddress(conf); + SecurityUtil.login(conf, YarnConfiguration.TIMELINE_SERVICE_KEYTAB, + YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, socAddr.getHostName()); + } + + /** + * Retrieve the timeline server bind address from configuration + * + * @param conf + * @return InetSocketAddress + */ + private static InetSocketAddress getBindAddress(Configuration conf) { + return conf.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/security/TimelineAuthenticationFilter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/security/TimelineAuthenticationFilter.java new file mode 100644 index 0000000..6765367 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/security/TimelineAuthenticationFilter.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.security; + +import java.util.Properties; + +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; + +@Private +@Unstable +public class TimelineAuthenticationFilter extends AuthenticationFilter { + + @Override + protected Properties getConfiguration(String configPrefix, + FilterConfig filterConfig) throws ServletException { + Properties properties = super.getConfiguration(configPrefix, filterConfig); + if (properties.getProperty(AUTH_TYPE).equals("kerberos")) { + properties.setProperty( + AUTH_TYPE, TimelineKerberosAuthenticationHandler.class.getName()); + } + return properties; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/security/TimelineDelegationTokenSecretManagerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/security/TimelineDelegationTokenSecretManagerService.java new file mode 100644 index 0000000..9cf5174 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/security/TimelineDelegationTokenSecretManagerService.java @@ -0,0 +1,189 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.security; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; + +/** + * The service wrapper of {@link TimelineDelegationTokenSecretManager} + */ +@Private +@Unstable +public class TimelineDelegationTokenSecretManagerService extends AbstractService { + + private TimelineDelegationTokenSecretManager secretManager = null; + private InetSocketAddress serviceAddr = null; + + public TimelineDelegationTokenSecretManagerService() { + super(TimelineDelegationTokenSecretManagerService.class.getName()); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + long secretKeyInterval = + conf.getLong(YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_KEY, + YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT); + long tokenMaxLifetime = + conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY, + YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); + long tokenRenewInterval = + conf.getLong(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_KEY, + YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); + secretManager = new TimelineDelegationTokenSecretManager(secretKeyInterval, + tokenMaxLifetime, tokenRenewInterval, + 3600000); + secretManager.startThreads(); + + if (YarnConfiguration.useHttps(getConfig())) { + serviceAddr = getConfig().getSocketAddr( + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT); + } else { + serviceAddr = getConfig().getSocketAddr( + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT); + } + super.init(conf); + } + + @Override + protected void serviceStop() throws Exception { + secretManager.stopThreads(); + super.stop(); + } + + /** + * Creates a delegation token. + * + * @param ugi UGI creating the token. + * @param renewer token renewer. + * @return new delegation token. + * @throws IOException thrown if the token could not be created. + */ + public Token createToken( + UserGroupInformation ugi, String renewer) throws IOException { + renewer = (renewer == null) ? ugi.getShortUserName() : renewer; + String user = ugi.getUserName(); + Text owner = new Text(user); + Text realUser = null; + if (ugi.getRealUser() != null) { + realUser = new Text(ugi.getRealUser().getUserName()); + } + TimelineDelegationTokenIdentifier tokenIdentifier = + new TimelineDelegationTokenIdentifier(owner, new Text(renewer), realUser); + Token token = + new Token(tokenIdentifier, secretManager); + SecurityUtil.setTokenService(token, serviceAddr); + return token; + } + + /** + * Renews a delegation token. + * + * @param token delegation token to renew. + * @param renewer token renewer. + * @throws IOException thrown if the token could not be renewed. + */ + public long renewToken(Token token, + String renewer) throws IOException { + return secretManager.renewToken(token, renewer); + } + + /** + * Cancels a delegation token. + * + * @param token delegation token to cancel. + * @param canceler token canceler. + * @throws IOException thrown if the token could not be canceled. + */ + public void cancelToken(Token token, + String canceler) throws IOException { + secretManager.cancelToken(token, canceler); + } + + /** + * Verifies a delegation token. + * + * @param token delegation token to verify. + * @return the UGI for the token. + * @throws IOException thrown if the token could not be verified. + */ + public UserGroupInformation verifyToken(Token token) + throws IOException { + ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); + DataInputStream dis = new DataInputStream(buf); + TimelineDelegationTokenIdentifier id = new TimelineDelegationTokenIdentifier(); + try { + id.readFields(dis); + secretManager.verifyToken(id, token.getPassword()); + } finally { + dis.close(); + } + return id.getUser(); + } + + /** + * Create a timeline secret manager + * + * @param delegationKeyUpdateInterval + * the number of seconds for rolling new secret keys. + * @param delegationTokenMaxLifetime + * the maximum lifetime of the delegation tokens + * @param delegationTokenRenewInterval + * how often the tokens must be renewed + * @param delegationTokenRemoverScanInterval + * how often the tokens are scanned for expired tokens + */ + @Private + @Unstable + public static class TimelineDelegationTokenSecretManager extends + AbstractDelegationTokenSecretManager { + + public TimelineDelegationTokenSecretManager(long delegationKeyUpdateInterval, + long delegationTokenMaxLifetime, long delegationTokenRenewInterval, + long delegationTokenRemoverScanInterval) { + super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, + delegationTokenRenewInterval, delegationTokenRemoverScanInterval); + } + + @Override + public TimelineDelegationTokenIdentifier createIdentifier() { + return new TimelineDelegationTokenIdentifier(); + } + + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/security/TimelineKerberosAuthenticationHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/security/TimelineKerberosAuthenticationHandler.java new file mode 100644 index 0000000..3366f65 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/security/TimelineKerberosAuthenticationHandler.java @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.security; + +import java.io.IOException; +import java.io.Writer; +import java.text.MessageFormat; +import java.util.HashSet; +import java.util.Set; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.MediaType; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.server.AuthenticationToken; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import org.apache.hadoop.yarn.security.client.TimelineKerberosAuthenticator; +import org.apache.hadoop.yarn.security.client.TimelineKerberosAuthenticator.DelegationTokenOperation; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.codehaus.jackson.map.ObjectMapper; + +/** + * Server side AuthenticationHandler that authenticates requests + * using the incoming delegation token as a 'delegation' query string parameter. + *

+ * If not delegation token is present in the request it delegates to the + * {@link KerberosAuthenticationHandler} + */ +@Private +@Unstable +public class TimelineKerberosAuthenticationHandler + extends KerberosAuthenticationHandler { + + public static final String TYPE = "kerberos-dt"; + private static final Set DELEGATION_TOKEN_OPS = new HashSet(); + private static final String OP_PARAM = "op"; + private static final String ENTER = System.getProperty("line.separator"); + + private ObjectMapper mapper; + + static { + DELEGATION_TOKEN_OPS.add( + DelegationTokenOperation.GETDELEGATIONTOKEN.toString()); + DELEGATION_TOKEN_OPS.add( + DelegationTokenOperation.RENEWDELEGATIONTOKEN.toString()); + DELEGATION_TOKEN_OPS.add( + DelegationTokenOperation.CANCELDELEGATIONTOKEN.toString()); + } + + public TimelineKerberosAuthenticationHandler() { + super(); + mapper = new ObjectMapper(); + YarnJacksonJaxbJsonProvider.configObjectMapper(mapper); + } + + /** + * Returns authentication type of the handler. + * + * @return delegationtoken-kerberos + */ + @Override + public String getType() { + return TYPE; + } + + @Override + public boolean managementOperation(AuthenticationToken token, + HttpServletRequest request, HttpServletResponse response) + throws IOException, AuthenticationException { + boolean requestContinues = true; + String op = request.getParameter(OP_PARAM); + op = (op != null) ? op.toUpperCase() : null; + if (DELEGATION_TOKEN_OPS.contains(op) && + !request.getMethod().equals("OPTIONS")) { + DelegationTokenOperation dtOp = + DelegationTokenOperation.valueOf(op); + if (dtOp.getHttpMethod().equals(request.getMethod())) { + if (dtOp.requiresKerberosCredentials() && token == null) { + response.sendError(HttpServletResponse.SC_UNAUTHORIZED, + MessageFormat.format( + "Operation [{0}] requires SPNEGO authentication established", + dtOp)); + requestContinues = false; + } else { + TimelineDelegationTokenSecretManagerService secretManager = + AHSWebApp.getInstance() + .getTimelineDelegationTokenSecretManagerService(); + try { + TimelineDelegationTokenResponse res = null; + switch (dtOp) { + case GETDELEGATIONTOKEN: + UserGroupInformation ownerUGI = + UserGroupInformation.createRemoteUser(token.getUserName()); + String renewerParam = + request + .getParameter(TimelineKerberosAuthenticator.RENEWER_PARAM); + if (renewerParam == null) { + renewerParam = token.getUserName(); + } + Token dToken = secretManager.createToken(ownerUGI, renewerParam); + res = new TimelineDelegationTokenResponse(); + res.setType(TimelineKerberosAuthenticator.DELEGATION_TOKEN_URL); + res.setContent(dToken.encodeToUrlString()); + break; + case RENEWDELEGATIONTOKEN: + case CANCELDELEGATIONTOKEN: + String tokenParam = + request + .getParameter(TimelineKerberosAuthenticator.TOKEN_PARAM); + if (tokenParam == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST, + MessageFormat.format( + "Operation [{0}] requires the parameter [{1}]", + dtOp, TimelineKerberosAuthenticator.TOKEN_PARAM)); + requestContinues = false; + } else { + if (dtOp == DelegationTokenOperation.CANCELDELEGATIONTOKEN) { + Token dt = + new Token(); + dt.decodeFromUrlString(tokenParam); + secretManager.cancelToken(dt, + UserGroupInformation.getCurrentUser().getUserName()); + } else { + Token dt = + new Token(); + dt.decodeFromUrlString(tokenParam); + long expirationTime = + secretManager.renewToken(dt, token.getUserName()); + res = new TimelineDelegationTokenResponse(); + res.setType(TimelineKerberosAuthenticator.DELEGATION_TOKEN_EXPIRATION_TIME); + res.setContent(expirationTime); + } + } + break; + } + if (requestContinues) { + response.setStatus(HttpServletResponse.SC_OK); + if (res != null) { + response.setContentType(MediaType.APPLICATION_JSON); + Writer writer = response.getWriter(); + mapper.writeValue(writer, res); + writer.write(ENTER); + writer.flush(); + + } + requestContinues = false; + } + } catch (IOException e) { + throw new AuthenticationException(e.toString(), e); + } + } + } else { + response + .sendError( + HttpServletResponse.SC_BAD_REQUEST, + MessageFormat + .format( + "Wrong HTTP method [{0}] for operation [{1}], it should be [{2}]", + request.getMethod(), dtOp, dtOp.getHttpMethod())); + requestContinues = false; + } + } + return requestContinues; + } + + /** + * Authenticates a request looking for the delegation + * query-string parameter and verifying it is a valid token. If there is not + * delegation query-string parameter, it delegates the + * authentication to the {@link KerberosAuthenticationHandler} unless it is + * disabled. + * + * @param request + * the HTTP client request. + * @param response + * the HTTP client response. + * + * @return the authentication token for the authenticated request. + * @throws IOException + * thrown if an IO error occurred. + * @throws AuthenticationException + * thrown if the authentication failed. + */ + @Override + public AuthenticationToken authenticate(HttpServletRequest request, + HttpServletResponse response) + throws IOException, AuthenticationException { + AuthenticationToken token; + String delegationParam = + request.getParameter(TimelineKerberosAuthenticator.DELEGATION_PARAM); + if (delegationParam != null) { + Token dt = + new Token(); + dt.decodeFromUrlString(delegationParam); + TimelineDelegationTokenSecretManagerService secretManager = + AHSWebApp.getInstance() + .getTimelineDelegationTokenSecretManagerService(); + UserGroupInformation ugi = secretManager.verifyToken(dt); + final String shortName = ugi.getShortUserName(); + // creating a ephemeral token + token = new AuthenticationToken(shortName, ugi.getUserName(), getType()); + token.setExpires(0); + } else { + token = super.authenticate(request, response); + } + return token; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java index 93065b3..6fbb2d1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java @@ -19,25 +19,70 @@ import static org.apache.hadoop.yarn.util.StringHelper.pajoin; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.server.api.ApplicationContext; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager; +import org.apache.hadoop.yarn.server.applicationhistoryservice.security.TimelineDelegationTokenSecretManagerService; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.YarnWebParams; +import com.google.common.annotations.VisibleForTesting; + public class AHSWebApp extends WebApp implements YarnWebParams { - private final ApplicationHistoryManager applicationHistoryManager; - private final TimelineStore timelineStore; + private ApplicationHistoryManager applicationHistoryManager; + private TimelineStore timelineStore; + private TimelineDelegationTokenSecretManagerService secretManagerService; + + private static AHSWebApp instance = null; + + public static AHSWebApp getInstance() { + if (instance == null) { + instance = new AHSWebApp(); + } + return instance; + } + + @Private + @VisibleForTesting + public static void resetInstance() { + instance = null; + } + + private AHSWebApp() { - public AHSWebApp(ApplicationHistoryManager applicationHistoryManager, - TimelineStore timelineStore) { + } + + public ApplicationHistoryManager getApplicationHistoryManager() { + return applicationHistoryManager; + } + + public void setApplicationHistoryManager( + ApplicationHistoryManager applicationHistoryManager) { this.applicationHistoryManager = applicationHistoryManager; + } + + public TimelineStore getTimelineStore() { + return timelineStore; + } + + public void setTimelineStore(TimelineStore timelineStore) { this.timelineStore = timelineStore; } + public TimelineDelegationTokenSecretManagerService + getTimelineDelegationTokenSecretManagerService() { + return secretManagerService; + } + + public void setTimelineDelegationTokenSecretManagerService( + TimelineDelegationTokenSecretManagerService secretManagerService) { + this.secretManagerService = secretManagerService; + } + @Override public void setup() { bind(YarnJacksonJaxbJsonProvider.class); @@ -46,6 +91,8 @@ public void setup() { bind(GenericExceptionHandler.class); bind(ApplicationContext.class).toInstance(applicationHistoryManager); bind(TimelineStore.class).toInstance(timelineStore); + bind(TimelineDelegationTokenSecretManagerService.class).toInstance( + secretManagerService); route("/", AHSController.class); route(pajoin("/apps", APP_STATE), AHSController.class); route(pajoin("/app", APPLICATION_ID), AHSController.class, "app"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java index a35fe46..3f3c08a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.junit.After; import org.junit.Before; @@ -74,6 +75,7 @@ public void setup() { @After public void tearDown() throws Exception { + AHSWebApp.resetInstance(); historyServer.stop(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java index d6d20af..5c55bec 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java @@ -26,6 +26,7 @@ import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.junit.After; import org.junit.Test; @@ -40,7 +41,7 @@ public void testStartStopServer() throws Exception { Configuration config = new YarnConfiguration(); historyServer.init(config); assertEquals(STATE.INITED, historyServer.getServiceState()); - assertEquals(3, historyServer.getServices().size()); + assertEquals(4, historyServer.getServices().size()); ApplicationHistoryClientService historyService = historyServer.getClientService(); assertNotNull(historyServer.getClientService()); @@ -73,5 +74,6 @@ public void stop() { if (historyServer != null) { historyServer.stop(); } + AHSWebApp.resetInstance(); } }