diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java index 37c0046..5eb6741 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelinePutResponse.java @@ -107,6 +107,12 @@ public void setErrors(List errors) { */ public static final int IO_EXCEPTION = 2; + /** + * Error code returned if the user specify the timeline system reserved + * a filter key + */ + public static final int SYSTEM_FILTER_CONFLICT = 3; + private String entityId; private String entityType; private int errorCode; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index 731ae14..50989b9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -33,8 +33,9 @@ import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; @@ -52,9 +53,10 @@ private static final Log LOG = LogFactory .getLog(ApplicationHistoryServer.class); - ApplicationHistoryClientService ahsClientService; - ApplicationHistoryManager historyManager; - TimelineStore timelineStore; + private ApplicationHistoryClientService ahsClientService; + private ApplicationHistoryManager historyManager; + private TimelineStore timelineStore; + private TimelineACLsManager timelineACLsManager; private WebApp webApp; public ApplicationHistoryServer() { @@ -69,6 +71,7 @@ protected void serviceInit(Configuration conf) throws Exception { addService((Service) historyManager); timelineStore = createTimelineStore(conf); addIfService(timelineStore); + timelineACLsManager = createTimelineACLsManager(conf); super.serviceInit(conf); } @@ -148,6 +151,10 @@ protected TimelineStore createTimelineStore( TimelineStore.class), conf); } + protected TimelineACLsManager createTimelineACLsManager(Configuration conf) { + return new TimelineACLsManager(conf); + } + protected void startWebApp() { String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(getConfig()); LOG.info("Instantiating AHSWebApp at " + bindAddress); @@ -162,7 +169,7 @@ protected void startWebApp() { .withHttpSpnegoKeytabKey( YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) .at(bindAddress) - .start(new AHSWebApp(historyManager, timelineStore)); + .start(new AHSWebApp(historyManager, timelineStore, timelineACLsManager)); } catch (Exception e) { String msg = "AHSWebApp failed to start."; LOG.error(msg, e); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java index 6b50d83..d5b5a00 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TimelineStore.java @@ -26,4 +26,15 @@ @InterfaceStability.Unstable public interface TimelineStore extends Service, TimelineReader, TimelineWriter { + + /** + * The system filter which will be automatically added to a + * {@link TimelineEntity}'s primary filter section when storing the entity. + * The filter key is case sensitive. Users are supposed not to use the key + * reserved by the timeline system. + */ + enum TimelineSystemFilter { + TIMELINE_SYSTEM_FILTER_OWNER + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java new file mode 100644 index 0000000..b3c637e --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineACLsManager.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; + +import java.io.IOException; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.EntityIdentifier; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore.TimelineSystemFilter; + +/** + * TimelineACLsManager check the entity level timeline data access. + */ +@Private +public class TimelineACLsManager { + + private static final Log LOG = LogFactory.getLog(TimelineACLsManager.class); + + private boolean aclsEnabled; + + public TimelineACLsManager(Configuration conf) { + aclsEnabled = conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE, + YarnConfiguration.DEFAULT_YARN_ACL_ENABLE); + } + + public boolean checkAccess(String user, + TimelineEntity entity) throws YarnException, IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Verifying the access of " + user + + " on the timeline entity " + + new EntityIdentifier(entity.getEntityId(), entity.getEntityType())); + } + + if (!aclsEnabled) { + return true; + } + + Set values = + entity.getPrimaryFilters().get( + TimelineSystemFilter.TIMELINE_SYSTEM_FILTER_OWNER.toString()); + if (values == null || values.size() != 1) { + throw new YarnException("Owner information of the timeline entity " + + new EntityIdentifier(entity.getEntityId(), entity.getEntityType()) + + " is corrupted."); + } + String owner = values.iterator().next().toString(); + // TODO: Currently we just check the user is the timeline entity owner. In + // the future, we need to check whether the user is admin or is in the + // allowed user/group list + if (user.equals(owner)) { + return true; + } + return false; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java index 93065b3..16a9002 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java @@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.server.api.ApplicationContext; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; @@ -31,11 +32,13 @@ private final ApplicationHistoryManager applicationHistoryManager; private final TimelineStore timelineStore; + private final TimelineACLsManager timelineAClsManager; public AHSWebApp(ApplicationHistoryManager applicationHistoryManager, - TimelineStore timelineStore) { + TimelineStore timelineStore, TimelineACLsManager timelineAClsManager) { this.applicationHistoryManager = applicationHistoryManager; this.timelineStore = timelineStore; + this.timelineAClsManager = timelineAClsManager; } @Override @@ -46,6 +49,7 @@ public void setup() { bind(GenericExceptionHandler.class); bind(ApplicationContext.class).toInstance(applicationHistoryManager); bind(TimelineStore.class).toInstance(timelineStore); + bind(TimelineACLsManager.class).toInstance(timelineAClsManager); route("/", AHSController.class); route(pajoin("/apps", APP_STATE), AHSController.class); route(pajoin("/app", APPLICATION_ID), AHSController.class, "app"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java index 44567ea..052dbcc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java @@ -25,6 +25,7 @@ import java.util.Collection; import java.util.EnumSet; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.SortedSet; @@ -52,15 +53,18 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.EntityIdentifier; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.NameValuePair; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineReader.Field; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; @@ -75,10 +79,13 @@ private static final Log LOG = LogFactory.getLog(TimelineWebServices.class); private TimelineStore store; + private TimelineACLsManager timelineACLsManager; @Inject - public TimelineWebServices(TimelineStore store) { + public TimelineWebServices(TimelineStore store, + TimelineACLsManager timelineACLsManager) { this.store = store; + this.timelineACLsManager = timelineACLsManager; } @XmlRootElement(name = "about") @@ -141,6 +148,13 @@ public TimelineEntities getEntities( init(res); TimelineEntities entities = null; try { + EnumSet fieldEnums = parseFieldsStr(fields, ","); + boolean extendFields = false; + if (!fieldEnums.contains(Field.PRIMARY_FILTERS)) { + fieldEnums.add(Field.PRIMARY_FILTERS); + extendFields = true; + } + String user = getUser(req); entities = store.getEntities( parseStr(entityType), parseLongStr(limit), @@ -151,6 +165,24 @@ public TimelineEntities getEntities( parsePairStr(primaryFilter, ":"), parsePairsStr(secondaryFilter, ",", ":"), parseFieldsStr(fields, ",")); + if (entities != null) { + Iterator entitiesItr = + entities.getEntities().iterator(); + while (entitiesItr.hasNext()) { + TimelineEntity entity = entitiesItr.next(); + // check ACLs + if (!timelineACLsManager.checkAccess(user, entity)) { + entitiesItr.remove(); + } else { + // clean up system data + if (extendFields) { + entity.setPrimaryFilters(null); + } else { + cleanupOwnerInfo(entity); + } + } + } + } } catch (NumberFormatException e) { throw new BadRequestException( "windowStart, windowEnd or limit is not a numeric value."); @@ -160,6 +192,10 @@ public TimelineEntities getEntities( LOG.error("Error getting entities", e); throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); + } catch (YarnException e) { + LOG.error("Error getting entities", e); + throw new WebApplicationException(e, + Response.Status.INTERNAL_SERVER_ERROR); } if (entities == null) { return new TimelineEntities(); @@ -182,9 +218,27 @@ public TimelineEntity getEntity( init(res); TimelineEntity entity = null; try { + EnumSet fieldEnums = parseFieldsStr(fields, ","); + boolean extendFields = false; + if (!fieldEnums.contains(Field.PRIMARY_FILTERS)) { + fieldEnums.add(Field.PRIMARY_FILTERS); + extendFields = true; + } entity = store.getEntity(parseStr(entityId), parseStr(entityType), - parseFieldsStr(fields, ",")); + fieldEnums); + // check ACLs + String user = getUser(req); + if (!timelineACLsManager.checkAccess(user, entity)) { + entity = null; + } else { + // clean up the system data + if (extendFields) { + entity.setPrimaryFilters(null); + } else { + cleanupOwnerInfo(entity); + } + } } catch (IllegalArgumentException e) { throw new BadRequestException( "requested invalid field."); @@ -192,6 +246,10 @@ public TimelineEntity getEntity( LOG.error("Error getting entity", e); throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); + } catch (YarnException e) { + LOG.error("Error getting entity", e); + throw new WebApplicationException(e, + Response.Status.INTERNAL_SERVER_ERROR); } if (entity == null) { throw new WebApplicationException(Response.Status.NOT_FOUND); @@ -217,6 +275,7 @@ public TimelineEvents getEvents( init(res); TimelineEvents events = null; try { + String user = getUser(req); events = store.getEntityTimelines( parseStr(entityType), parseArrayStr(entityId, ","), @@ -224,6 +283,21 @@ public TimelineEvents getEvents( parseLongStr(windowStart), parseLongStr(windowEnd), parseArrayStr(eventType, ",")); + if (events != null) { + Iterator eventsItr = + events.getAllEvents().iterator(); + while (eventsItr.hasNext()) { + TimelineEvents.EventsOfOneEntity eventsOfOneEntity = eventsItr.next(); + TimelineEntity entity = store.getEntity( + eventsOfOneEntity.getEntityId(), + eventsOfOneEntity.getEntityType(), + EnumSet.of(Field.PRIMARY_FILTERS)); + // check ACLs + if (!timelineACLsManager.checkAccess(user, entity)) { + eventsItr.remove(); + } + } + } } catch (NumberFormatException e) { throw new BadRequestException( "windowStart, windowEnd or limit is not a numeric value."); @@ -231,6 +305,10 @@ public TimelineEvents getEvents( LOG.error("Error getting entity timelines", e); throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); + } catch (YarnException e) { + LOG.error("Error getting entity timelines", e); + throw new WebApplicationException(e, + Response.Status.INTERNAL_SERVER_ERROR); } if (events == null) { return new TimelineEvents(); @@ -252,12 +330,33 @@ public TimelinePutResponse postEntities( if (entities == null) { return new TimelinePutResponse(); } + String user = getUser(req); try { List entityIDs = new ArrayList(); + TimelineEntities entitiesToPut = new TimelineEntities(); + List errors = + new ArrayList(); for (TimelineEntity entity : entities.getEntities()) { EntityIdentifier entityID = new EntityIdentifier(entity.getEntityId(), entity.getEntityType()); + // inject owner information for the access check + try { + injectOwnerInfo(entity, user); + } catch (YarnException e) { + // Skip the entity which messes up the primary filter and record the + // error + LOG.warn("Skip the timeline entity: " + entityID, e); + TimelinePutResponse.TimelinePutError error = + new TimelinePutResponse.TimelinePutError(); + error.setEntityId(entityID.getId()); + error.setEntityType(entityID.getType()); + error.setErrorCode( + TimelinePutResponse.TimelinePutError.SYSTEM_FILTER_CONFLICT); + errors.add(error); + continue; + } entityIDs.add(entityID); + entitiesToPut.addEntity(entity); if (LOG.isDebugEnabled()) { LOG.debug("Storing the entity " + entityID + ", JSON-style content: " + TimelineUtils.dumpTimelineRecordtoJSON(entity)); @@ -266,7 +365,10 @@ public TimelinePutResponse postEntities( if (LOG.isDebugEnabled()) { LOG.debug("Storing entities: " + CSV_JOINER.join(entityIDs)); } - return store.put(entities); + TimelinePutResponse response = store.put(entitiesToPut); + // add the errors of timeline system filter key conflict + response.addErrors(errors); + return response; } catch (IOException e) { LOG.error("Error putting entities", e); throw new WebApplicationException(e, @@ -358,4 +460,34 @@ private static String parseStr(String str) { return str == null ? null : str.trim(); } + private static String getUser(HttpServletRequest req) { + String remoteUser = req.getRemoteUser(); + UserGroupInformation callerUGI = null; + if (remoteUser != null) { + callerUGI = UserGroupInformation.createRemoteUser(remoteUser); + } + return callerUGI == null ? "" : callerUGI.getShortUserName(); + } + + private static void injectOwnerInfo(TimelineEntity timelineEntity, + String owner) throws YarnException { + if (timelineEntity.getPrimaryFilters() != null && + timelineEntity.getPrimaryFilters().containsKey( + TimelineStore.TimelineSystemFilter.TIMELINE_SYSTEM_FILTER_OWNER)) { + throw new YarnException( + "User should not use the timeline system filter key: " + + TimelineStore.TimelineSystemFilter.TIMELINE_SYSTEM_FILTER_OWNER); + } + timelineEntity.addPrimaryFilter( + TimelineStore.TimelineSystemFilter.TIMELINE_SYSTEM_FILTER_OWNER + .toString(), owner); + } + + private static void cleanupOwnerInfo(TimelineEntity timelineEntity) { + if (timelineEntity.getPrimaryFilters() != null) { + timelineEntity.getPrimaryFilters().remove( + TimelineStore.TimelineSystemFilter.TIMELINE_SYSTEM_FILTER_OWNER); + } + } + }