diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineNamespace.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineNamespace.java new file mode 100644 index 0000000..da87211 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineNamespace.java @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.timeline; + +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + *

+ * This class contains the information about a timeline namespace, which is used + * to a user to host a number of timeline entities, isolating them from others'. + * The user can also define the reader and writer users/groups for the the + * namespace, which is used to control the access to its entities. + *

+ * + *

+ * The reader and writer users/groups pattern that the user can supply is the + * same as what AccessControlList takes. + *

+ * + */ +@XmlRootElement(name = "namespace") +@XmlAccessorType(XmlAccessType.NONE) +@Public +@Unstable +public class TimelineNamespace { + + private String id; + private String description; + private String owner; + private List readers = new ArrayList(); + private List writers = new ArrayList(); + private Long createdTime; + private Long modifiedTime; + + public TimelineNamespace() { + } + + /** + * Get the namespace ID + * + * @return the namespace ID + */ + @XmlElement(name = "id") + public String getId() { + return id; + } + + /** + * Set the namespace ID + * + * @param id the namespace ID + */ + public void setId(String id) { + this.id = id; + } + + /** + * Get the namespace description + * + * @return the namespace description + */ + @XmlElement(name = "description") + public String getDescription() { + return description; + } + + /** + * Set the namespace description + * + * @param description the namespace description + */ + public void setDescription(String description) { + this.description = description; + } + + /** + * Get the namespace owner + * + * @return the namespace owner + */ + @XmlElement(name = "owner") + public String getOwner() { + return owner; + } + + /** + * Set the namespace owner. The user doesn't need to set it, which will + * automatically set to the user who puts the namespace. + * + * @param owner the namespace owner + */ + public void setOwner(String owner) { + this.owner = owner; + } + + /** + * Get the reader list + * + * @return the reader list + */ + @XmlElement(name = "readers") + public List getReaders() { + return readers; + } + + /** + * Set the reader list to the given list of readers + * + * @param readers the reader list + */ + public void setReaders(List readers) { + this.readers = readers; + } + + /** + * Add a list of readers to the existing reader list + * + * @param readers the reader list + */ + public void addReaders(List readers) { + this.readers.addAll(readers); + } + + /** + * Add a single reader to the existing reader list + * + * @param reader the reader + */ + public void addReader(String reader) { + readers.add(reader); + } + + /** + * Get the writer list + * + * @return the writer list + */ + @XmlElement(name = "writers") + public List getWriters() { + return writers; + } + + /** + * Set the writer list to the given list of writers + * + * @param writers the writer list + */ + public void setWriters(List writers) { + this.writers = writers; + } + + /** + * Add a list of writers to the existing writer list + * + * @param writers the writer list + */ + public void addWriters(List writers) { + this.writers.addAll(writers); + } + + /** + * Add a single writer to the existing writer list + * + * @param writer the writer + */ + public void addWriter(String writer) { + writers.add(writer); + } + + /** + * Get the created time of the namespace + * + * @return the created time of the namespace + */ + @XmlElement(name = "createdtime") + public Long getCreatedTime() { + return createdTime; + } + + /** + * Set the created time of the namespace + * + * @param createdTime the created time of the namespace + */ + public void setCreatedTime(Long createdTime) { + this.createdTime = createdTime; + } + + /** + * Get the modified time of the namespace + * + * @return the modified time of the namespace + */ + @XmlElement(name = "modifiedtime") + public Long getModifiedTime() { + return modifiedTime; + } + + /** + * Set the modified time of the namespace + * + * @param modifiedTime the modified time of the namespace + */ + public void setModifiedTime(Long modifiedTime) { + this.modifiedTime = modifiedTime; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineNamespaces.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineNamespaces.java new file mode 100644 index 0000000..63c08a6 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineNamespaces.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.timeline; + +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * The class that hosts a list of timeline namespaces. + */ +@XmlRootElement(name = "namespaces") +@XmlAccessorType(XmlAccessType.NONE) +@Public +@Unstable +public class TimelineNamespaces { + + private List namespaces = + new ArrayList(); + + public TimelineNamespaces() { + } + + /** + * Get a list of namespaces + * + * @return a list of namespaces + */ + @XmlElement(name = "namespaces") + public List getNamespaces() { + return namespaces; + } + + /** + * Add a single namespace into the existing namespace list + * + * @param namespace + * a single namespace + */ + public void addEntity(TimelineNamespace namespace) { + namespaces.add(namespace); + } + + /** + * All a list of namespaces into the existing namespace list + * + * @param namespaces + * a list of namespaces + */ + public void addEntities(List namespaces) { + this.namespaces.addAll(namespaces); + } + + /** + * Set the namespace list to the given list of namespaces + * + * @param namespaces + * a list of namespaces + */ + public void setNamespaces(List namespaces) { + this.namespaces = namespaces; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java index de1d3e2..fa86be3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java @@ -26,6 +26,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -69,6 +70,22 @@ public abstract TimelinePutResponse putEntities( /** *

+ * Send the information of a namespace to the timeline server. It is a + * blocking API. The method will not return until it gets the response from + * the timeline server. + *

+ * + * @param namespace + * an {@link TimelineNamespace} object + * @throws IOException + * @throws YarnException + */ + @Public + public abstract void putNamespace( + TimelineNamespace namespace) throws IOException, YarnException; + + /** + *

* Get a delegation token so as to be able to talk to the timeline server in a * secure way. *

diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index f383a8a..dbfed54 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -52,6 +52,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -150,9 +151,27 @@ public TimelinePutResponse putEntities( } TimelineEntities entitiesContainer = new TimelineEntities(); entitiesContainer.addEntities(Arrays.asList(entities)); + ClientResponse resp = doPosting(entitiesContainer, null); + return resp.getEntity(TimelinePutResponse.class); + } + + + @Override + public void putNamespace(TimelineNamespace namespace) throws IOException, + YarnException { + if (!isEnabled) { + if (LOG.isDebugEnabled()) { + LOG.debug("Nothing will be put because timeline service is not enabled"); + } + return; + } + doPosting(namespace, "namespace"); + } + + private ClientResponse doPosting(Object obj, String path) throws IOException, YarnException { ClientResponse resp; try { - resp = doPostingEntities(entitiesContainer); + resp = doPostingObject(obj, path); } catch (RuntimeException re) { // runtime exception is expected if the client cannot connect the server String msg = @@ -172,7 +191,7 @@ public TimelinePutResponse putEntities( } throw new YarnException(msg); } - return resp.getEntity(TimelinePutResponse.class); + return resp; } @Override @@ -184,11 +203,14 @@ public TimelinePutResponse putEntities( @Private @VisibleForTesting - public ClientResponse doPostingEntities(TimelineEntities entities) { + public ClientResponse doPostingObject(Object object, String path) { WebResource webResource = client.resource(resURI); + if (path != null) { + webResource.path(path); + } return webResource.accept(MediaType.APPLICATION_JSON) .type(MediaType.APPLICATION_JSON) - .post(ClientResponse.class, entities); + .post(ClientResponse.class, object); } private static class PseudoAuthenticatedURLConnectionFactory diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java index 7813b5d..4a96796 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java @@ -161,4 +161,32 @@ public void testTimelinePutErrors() throws Exception { Assert.assertEquals(error2.getErrorCode(), e.getErrorCode()); } + @Test + public void testTimelineNamespace() throws Exception { + TimelineNamespace namespace = new TimelineNamespace(); + namespace.setId("test id"); + namespace.setDescription("test description"); + namespace.setOwner("test owner"); + namespace.addReader("test reader user"); + namespace.addReader("test reader group"); + namespace.addWriter("test writer user"); + namespace.addWriter("test writer group"); + namespace.setCreatedTime(0L); + namespace.setModifiedTime(1L); + LOG.info("Namespace in JSON:"); + LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(namespace, true)); + + Assert.assertEquals("test id", namespace.getId()); + Assert.assertEquals("test description", namespace.getDescription()); + Assert.assertEquals("test owner", namespace.getOwner()); + Assert.assertEquals(2, namespace.getReaders().size()); + Assert.assertEquals("test reader user", namespace.getReaders().get(0)); + Assert.assertEquals("test reader group", namespace.getReaders().get(1)); + Assert.assertEquals(2, namespace.getWriters().size()); + Assert.assertEquals("test writer user", namespace.getWriters().get(0)); + Assert.assertEquals("test writer group", namespace.getWriters().get(1)); + Assert.assertEquals(new Long(0L), namespace.getCreatedTime()); + Assert.assertEquals(new Long(1L), namespace.getModifiedTime()); + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java index 3c5272a..7727de7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java @@ -27,16 +27,16 @@ import java.net.ConnectException; -import org.junit.Assert; - import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -63,7 +63,7 @@ public void tearDown() { @Test public void testPostEntities() throws Exception { - mockClientResponse(client, ClientResponse.Status.OK, false, false); + mockEntityClientResponse(client, ClientResponse.Status.OK, false, false); try { TimelinePutResponse response = client.putEntities(generateEntity()); Assert.assertEquals(0, response.getErrors().size()); @@ -74,7 +74,7 @@ public void testPostEntities() throws Exception { @Test public void testPostEntitiesWithError() throws Exception { - mockClientResponse(client, ClientResponse.Status.OK, true, false); + mockEntityClientResponse(client, ClientResponse.Status.OK, true, false); try { TimelinePutResponse response = client.putEntities(generateEntity()); Assert.assertEquals(1, response.getErrors().size()); @@ -91,7 +91,7 @@ public void testPostEntitiesWithError() throws Exception { @Test public void testPostEntitiesNoResponse() throws Exception { - mockClientResponse( + mockEntityClientResponse( client, ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false); try { client.putEntities(generateEntity()); @@ -104,7 +104,7 @@ public void testPostEntitiesNoResponse() throws Exception { @Test public void testPostEntitiesConnectionRefused() throws Exception { - mockClientResponse(client, null, false, true); + mockEntityClientResponse(client, null, false, true); try { client.putEntities(generateEntity()); Assert.fail("RuntimeException is expected"); @@ -118,7 +118,7 @@ public void testPostEntitiesTimelineServiceNotEnabled() throws Exception { YarnConfiguration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); TimelineClientImpl client = createTimelineClient(conf); - mockClientResponse( + mockEntityClientResponse( client, ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false); try { TimelinePutResponse response = client.putEntities(generateEntity()); @@ -137,7 +137,7 @@ public void testPostEntitiesTimelineServiceDefaultNotEnabled() // Make sure default value is pickup up conf.unset(YarnConfiguration.TIMELINE_SERVICE_ENABLED); TimelineClientImpl client = createTimelineClient(conf); - mockClientResponse(client, ClientResponse.Status.INTERNAL_SERVER_ERROR, + mockEntityClientResponse(client, ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false); try { TimelinePutResponse response = client.putEntities(generateEntity()); @@ -148,16 +148,50 @@ public void testPostEntitiesTimelineServiceDefaultNotEnabled() } } - private static ClientResponse mockClientResponse(TimelineClientImpl client, - ClientResponse.Status status, boolean hasError, boolean hasRuntimeError) { + @Test + public void testPutNamespace() throws Exception { + mockNamespaceClientResponse(client, ClientResponse.Status.OK, false); + try { + client.putNamespace(generateNamespace()); + } catch (YarnException e) { + Assert.fail("Exception is not expected"); + } + } + + @Test + public void testPutNamespaceNoResponse() throws Exception { + mockNamespaceClientResponse(client, ClientResponse.Status.FORBIDDEN, false); + try { + client.putNamespace(generateNamespace()); + Assert.fail("Exception is expected"); + } catch (YarnException e) { + Assert.assertTrue(e.getMessage().contains( + "Failed to get the response from the timeline server.")); + } + } + + @Test + public void testPutNamespaceConnectionRefused() throws Exception { + mockNamespaceClientResponse(client, null, true); + try { + client.putNamespace(generateNamespace()); + Assert.fail("RuntimeException is expected"); + } catch (RuntimeException re) { + Assert.assertTrue(re instanceof ClientHandlerException); + } + } + + private static ClientResponse mockEntityClientResponse( + TimelineClientImpl client, ClientResponse.Status status, + boolean hasError, boolean hasRuntimeError) { ClientResponse response = mock(ClientResponse.class); if (hasRuntimeError) { doThrow(new ClientHandlerException(new ConnectException())).when(client) - .doPostingEntities(any(TimelineEntities.class)); + .doPostingObject(any(TimelineEntities.class), any(String.class)); return response; } doReturn(response).when(client) - .doPostingEntities(any(TimelineEntities.class)); + .doPostingObject(any(TimelineEntities.class), any(String.class)); when(response.getClientResponseStatus()).thenReturn(status); TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError(); @@ -172,6 +206,21 @@ private static ClientResponse mockClientResponse(TimelineClientImpl client, return response; } + private static ClientResponse mockNamespaceClientResponse( + TimelineClientImpl client, ClientResponse.Status status, + boolean hasRuntimeError) { + ClientResponse response = mock(ClientResponse.class); + if (hasRuntimeError) { + doThrow(new ClientHandlerException(new ConnectException())).when(client) + .doPostingObject(any(TimelineNamespace.class), any(String.class)); + return response; + } + doReturn(response).when(client) + .doPostingObject(any(TimelineNamespace.class), any(String.class)); + when(response.getClientResponseStatus()).thenReturn(status); + return response; + } + private static TimelineEntity generateEntity() { TimelineEntity entity = new TimelineEntity(); entity.setEntityId("entity id"); @@ -194,6 +243,18 @@ private static TimelineEntity generateEntity() { return entity; } + public static TimelineNamespace generateNamespace() { + TimelineNamespace namespace = new TimelineNamespace(); + namespace.setId("namesapce id"); + namespace.setDescription("namespace description"); + namespace.setOwner("namespace owner"); + namespace.addReader("namespace reader"); + namespace.addWriter("namespace writer"); + namespace.setCreatedTime(0L); + namespace.setModifiedTime(1L); + return namespace; + } + private static TimelineClientImpl createTimelineClient( YarnConfiguration conf) { TimelineClientImpl client = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java index b0feac1..5535922 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java @@ -58,6 +58,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespaces; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -145,6 +147,14 @@ private static final byte[] INVISIBLE_REVERSE_RELATED_ENTITIES_COLUMN = "z".getBytes(); + private static final byte[] NAMESPACE_ENTRY_PREFIX = "n".getBytes(); + private static final byte[] OWNER_LOOKUP_PREFIX = "o".getBytes(); + private static final byte[] DESCRIPTION_COLUMN = "d".getBytes(); + private static final byte[] OWNER_COLUMN = "o".getBytes(); + private static final byte[] READER_COLUMN = "r".getBytes(); + private static final byte[] WRITER_COLUMN = "w".getBytes(); + private static final byte[] TIMESTAMP_COLUMN = "t".getBytes(); + private static final byte[] EMPTY_BYTES = new byte[0]; private static final String TIMELINE_STORE_VERSION_KEY = "timeline-store-version"; @@ -1558,5 +1568,209 @@ private void checkVersion() throws IOException { throw new IOException(incompatibleMessage); } } - + + //TODO: make data retention work with the namespace data as well + @Override + public void put(TimelineNamespace namespace) throws IOException { + WriteBatch writeBatch = null; + try { + writeBatch = db.createWriteBatch(); + if (namespace.getId() == null || namespace.getId().length() == 0) { + throw new IllegalArgumentException("Namespace doesn't have an ID"); + } + if (namespace.getOwner() == null || namespace.getOwner().length() == 0) { + throw new IllegalArgumentException("Namespace doesn't have an owner."); + } + + // Write description + byte[] namespaceEntryKey = createNamespaceEntryKey( + namespace.getId(), DESCRIPTION_COLUMN); + byte[] ownerLookupEntryKey = createOwnerLookupKey( + namespace.getOwner(), namespace.getId(), DESCRIPTION_COLUMN); + if (namespace.getDescription() != null) { + writeBatch.put(namespaceEntryKey, namespace.getDescription().getBytes()); + writeBatch.put(ownerLookupEntryKey, namespace.getDescription().getBytes()); + } else { + writeBatch.put(namespaceEntryKey, EMPTY_BYTES); + writeBatch.put(ownerLookupEntryKey, EMPTY_BYTES); + } + + // Write owner + namespaceEntryKey = createNamespaceEntryKey(namespace.getId(), OWNER_COLUMN); + ownerLookupEntryKey = createOwnerLookupKey( + namespace.getOwner(), namespace.getId(), OWNER_COLUMN); + // Null check for owner is done before + writeBatch.put(namespaceEntryKey, namespace.getOwner().getBytes()); + writeBatch.put(ownerLookupEntryKey, namespace.getOwner().getBytes()); + + // Write readers + namespaceEntryKey = createNamespaceEntryKey(namespace.getId(), READER_COLUMN); + ownerLookupEntryKey = createOwnerLookupKey( + namespace.getOwner(), namespace.getId(), READER_COLUMN); + if (namespace.getReaders() != null && namespace.getReaders().size() > 0) { + byte[] readers = GenericObjectMapper.write(namespace.getReaders()); + writeBatch.put(namespaceEntryKey, readers); + writeBatch.put(ownerLookupEntryKey, readers); + } else { + writeBatch.put(namespaceEntryKey, EMPTY_BYTES); + writeBatch.put(ownerLookupEntryKey, EMPTY_BYTES); + } + + // Write writers + namespaceEntryKey = createNamespaceEntryKey(namespace.getId(), WRITER_COLUMN); + ownerLookupEntryKey = createOwnerLookupKey( + namespace.getOwner(), namespace.getId(), WRITER_COLUMN); + if (namespace.getWriters() != null && namespace.getWriters().size() > 0) { + byte[] writers = GenericObjectMapper.write(namespace.getWriters()); + writeBatch.put(namespaceEntryKey, writers); + writeBatch.put(ownerLookupEntryKey, writers); + } else { + writeBatch.put(namespaceEntryKey, EMPTY_BYTES); + writeBatch.put(ownerLookupEntryKey, EMPTY_BYTES); + } + + // Write creation time and modification time + namespaceEntryKey = createNamespaceEntryKey(namespace.getId(), TIMESTAMP_COLUMN); + ownerLookupEntryKey = createOwnerLookupKey( + namespace.getOwner(), namespace.getId(), TIMESTAMP_COLUMN); + long currentTimestamp = System.currentTimeMillis(); + byte[] timestamps = db.get(namespaceEntryKey); + if (timestamps == null) { + timestamps = new byte[16]; + writeReverseOrderedLong(currentTimestamp, timestamps, 0); + writeReverseOrderedLong(currentTimestamp, timestamps, 8); + } else { + writeReverseOrderedLong(currentTimestamp, timestamps, 8); + } + writeBatch.put(namespaceEntryKey, timestamps); + writeBatch.put(ownerLookupEntryKey, timestamps); + db.write(writeBatch); + } finally { + IOUtils.cleanup(LOG, writeBatch); + } + } + + /** + * Creates a namespace entity key with column name suffix, + * of the form NAMESPACE_ENTRY_PREFIX + namespace id + column name. + */ + private static byte[] createNamespaceEntryKey(String namespaceId, + byte[] columnName) throws IOException { + return KeyBuilder.newInstance().add(NAMESPACE_ENTRY_PREFIX) + .add(namespaceId).add(columnName).getBytes(); + } + + /** + * Creates an owner lookup key with column name suffix, + * of the form OWNER_LOOKUP_PREFIX + owner + namespace id + column name. + */ + private static byte[] createOwnerLookupKey( + String owner, String namespaceId, byte[] columnName) throws IOException { + return KeyBuilder.newInstance().add(OWNER_LOOKUP_PREFIX) + .add(owner).add(namespaceId).add(columnName).getBytes(); + } + + @Override + public TimelineNamespace getNamespace(String namespaceId) + throws IOException { + DBIterator iterator = null; + try { + byte[] prefix = KeyBuilder.newInstance() + .add(NAMESPACE_ENTRY_PREFIX).add(namespaceId).getBytesForLookup(); + iterator = db.iterator(); + iterator.seek(prefix); + return getTimelineNamespace(iterator, namespaceId, prefix); + } finally { + IOUtils.cleanup(LOG, iterator); + } + } + + @Override + public TimelineNamespaces getNamespaces(String owner) + throws IOException { + DBIterator iterator = null; + try { + byte[] prefix = KeyBuilder.newInstance() + .add(OWNER_LOOKUP_PREFIX).add(owner).getBytesForLookup(); + List namespaces = new ArrayList(); + for (iterator = db.iterator(), iterator.seek(prefix); + iterator.hasNext();) { + byte[] key = iterator.peekNext().getKey(); + if (!prefixMatches(prefix, prefix.length, key)) { + break; + } + // Iterator to parse the rows of an individual namespace + KeyParser kp = new KeyParser(key, prefix.length); + String namespaceId = kp.getNextString(); + byte[] prefixExt = KeyBuilder.newInstance().add(OWNER_LOOKUP_PREFIX) + .add(owner).add(namespaceId).getBytesForLookup(); + TimelineNamespace namespaceToReturn = + getTimelineNamespace(iterator, namespaceId, prefixExt); + if (namespaceToReturn != null) { + namespaces.add(namespaceToReturn); + } + } + // Sort the namespaces to return + Collections.sort(namespaces, new Comparator() { + @Override + public int compare( + TimelineNamespace namespace1, TimelineNamespace namespace2) { + int result = namespace2.getCreatedTime().compareTo( + namespace1.getCreatedTime()); + if (result == 0) { + return namespace2.getModifiedTime().compareTo( + namespace1.getModifiedTime()); + } else { + return result; + } + } + }); + TimelineNamespaces namespacesToReturn = new TimelineNamespaces(); + namespacesToReturn.addEntities(namespaces); + return namespacesToReturn; + } finally { + IOUtils.cleanup(LOG, iterator); + } + } + + @SuppressWarnings("unchecked") + private static TimelineNamespace getTimelineNamespace( + DBIterator iterator, String namespaceId, byte[] prefix) throws IOException { + // Iterate over all the rows whose key starts with prefix to retrieve the + // namespace information. + TimelineNamespace namespace = new TimelineNamespace(); + namespace.setId(namespaceId); + boolean noRows = true; + for (; iterator.hasNext(); iterator.next()) { + byte[] key = iterator.peekNext().getKey(); + if (!prefixMatches(prefix, prefix.length, key)) { + break; + } + if (noRows) { + noRows = false; + } + byte[] value = iterator.peekNext().getValue(); + if (value != null && value.length > 0) { + if (key[prefix.length] == DESCRIPTION_COLUMN[0]) { + namespace.setDescription(new String(value)); + } else if (key[prefix.length] == OWNER_COLUMN[0]) { + namespace.setOwner(new String(value)); + } else if (key[prefix.length] == READER_COLUMN[0]) { + namespace.addReaders( + (List)GenericObjectMapper.read(value)); + } else if (key[prefix.length] == WRITER_COLUMN[0]) { + namespace.addWriters( + (List)GenericObjectMapper.read(value)); + } else if (key[prefix.length] == TIMESTAMP_COLUMN[0]) { + namespace.setCreatedTime(readReverseOrderedLong(value, 0)); + namespace.setModifiedTime(readReverseOrderedLong(value, 8)); + } + } + } + if (noRows) { + return null; + } else { + return namespace; + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java index b94711c..ead40cf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java @@ -18,11 +18,14 @@ package org.apache.hadoop.yarn.server.timeline; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -40,6 +43,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespaces; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; @@ -59,6 +64,10 @@ new HashMap(); private Map entityInsertTimes = new HashMap(); + private Map namespacesById = + new HashMap(); + private Map> namespacesByOwner = + new HashMap>(); public MemoryTimelineStore() { super(MemoryTimelineStore.class.getName()); @@ -209,6 +218,58 @@ public TimelineEvents getEntityTimelines(String entityType, } @Override + public TimelineNamespace getNamespace(String namespaceId) + throws IOException { + TimelineNamespace namespace = namespacesById.get(namespaceId); + if (namespace == null) { + return null; + } else { + return createTimelineNamespace( + namespace.getId(), + namespace.getDescription(), + namespace.getOwner(), + namespace.getReaders(), + namespace.getWriters(), + namespace.getCreatedTime(), + namespace.getModifiedTime()); + } + } + + @Override + public TimelineNamespaces getNamespaces(String owner) + throws IOException { + List namespaces = new ArrayList(); + for (TimelineNamespace namespace : namespacesByOwner.get(owner)) { + TimelineNamespace namespaceToReturn = createTimelineNamespace( + namespace.getId(), + namespace.getDescription(), + namespace.getOwner(), + namespace.getReaders(), + namespace.getWriters(), + namespace.getCreatedTime(), + namespace.getModifiedTime()); + namespaces.add(namespaceToReturn); + } + Collections.sort(namespaces, new Comparator() { + @Override + public int compare( + TimelineNamespace namespace1, TimelineNamespace namespace2) { + int result = namespace2.getCreatedTime().compareTo( + namespace1.getCreatedTime()); + if (result == 0) { + return namespace2.getModifiedTime().compareTo( + namespace1.getModifiedTime()); + } else { + return result; + } + } + }); + TimelineNamespaces namespacesToReturn = new TimelineNamespaces(); + namespacesToReturn.addEntities(namespaces); + return namespacesToReturn; + } + + @Override public TimelinePutResponse put(TimelineEntities data) { TimelinePutResponse response = new TimelinePutResponse(); for (TimelineEntity entity : data.getEntities()) { @@ -306,6 +367,44 @@ public TimelinePutResponse put(TimelineEntities data) { return response; } + public void put(TimelineNamespace namespace) throws IOException { + TimelineNamespace namespaceToReplace = + namespacesById.get(namespace.getId()); + long currentTimestamp = System.currentTimeMillis(); + TimelineNamespace namespaceToStore = createTimelineNamespace( + namespace.getId(), namespace.getDescription(), namespace.getOwner(), + namespace.getReaders(), namespace.getWriters(), + (namespaceToReplace == null ? + currentTimestamp : namespaceToReplace.getCreatedTime()), + currentTimestamp); + namespacesById.put(namespaceToStore.getId(), namespaceToStore); + Set namespacesByOneOwner = + namespacesByOwner.get(namespaceToStore.getOwner()); + if (namespacesByOneOwner == null) { + namespacesByOneOwner = new HashSet(); + namespacesByOwner.put(namespaceToStore.getOwner(), namespacesByOneOwner); + } + if (namespaceToReplace != null) { + namespacesByOneOwner.remove(namespaceToReplace); + } + namespacesByOneOwner.add(namespaceToStore); + } + + private static TimelineNamespace createTimelineNamespace( + String id, String description, String owner, + List readers, List writers, + Long createdTime, Long modifiedTime) { + TimelineNamespace namespaceToStore = new TimelineNamespace(); + namespaceToStore.setId(id); + namespaceToStore.setDescription(description); + namespaceToStore.setOwner(owner); + namespaceToStore.addReaders(readers); + namespaceToStore.addWriters(writers); + namespaceToStore.setCreatedTime(createdTime); + namespaceToStore.setModifiedTime(modifiedTime); + return namespaceToStore; + } + private static TimelineEntity maskFields( TimelineEntity entity, EnumSet fields) { // Conceal the fields that are not going to be exposed diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java index e68e860..217fbe4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java @@ -34,6 +34,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespaces; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; @@ -286,6 +288,78 @@ public TimelinePutResponse postEntities( return response; } + /** + * Add or update an namespace. If the namespace already exists, only the owner + * and the admin can update it. + */ + public void putNamespace(TimelineNamespace namespace, + UserGroupInformation callerUGI) throws YarnException, IOException { + TimelineNamespace existingNamespace = + store.getNamespace(namespace.getId()); + if (existingNamespace != null) { + if (!timelineACLsManager.checkAccess(callerUGI, existingNamespace)) { + throw new YarnException(callerUGI.getShortUserName() + + " is not allowed to override an existing namespace " + + existingNamespace.getId()); + } + // Set it again in case ACLs are not enabled: The namespace can be + // modified by every body, but the owner is not changed. + namespace.setOwner(existingNamespace.getOwner()); + } + store.put(namespace); + } + + /** + * Get a single namespace of the particular ID. If callerUGI is not the owner + * or the admin of the namespace, we need to hide the details from him, and + * only allow him to see the ID. + */ + public TimelineNamespace getNamspace(String namespaceId, + UserGroupInformation callerUGI) throws YarnException, IOException { + TimelineNamespace namespace = store.getNamespace(namespaceId); + if (namespace != null) { + if (timelineACLsManager.checkAccess(callerUGI, namespace)) { + return namespace; + } else { + hideNamespaceDetails(namespace); + return namespace; + } + } + return null; + } + + /** + * Get all the namespaces that belong to the given owner. If callerUGI is not + * the owner or the admin of the namespace, we need to hide the details from + * him, and only allow him to see the ID. + */ + public TimelineNamespaces getNamespaces(String owner, + UserGroupInformation callerUGI) throws YarnException, IOException { + TimelineNamespaces namespaces = store.getNamespaces(owner); + boolean hasAccess = true; + boolean isChecked = false; + for (TimelineNamespace namespace : namespaces.getNamespaces()) { + // The owner for each namespace is the same, just need to check on + if (!isChecked) { + hasAccess = timelineACLsManager.checkAccess(callerUGI, namespace); + isChecked = true; + } + if (!hasAccess) { + hideNamespaceDetails(namespace); + } + } + return namespaces; + } + + private static void hideNamespaceDetails(TimelineNamespace namespace) { + namespace.setDescription(null); + namespace.setOwner(null); + namespace.setReaders(null); + namespace.setWriters(null); + namespace.setCreatedTime(null); + namespace.setModifiedTime(null); + } + private static boolean extendFields(EnumSet fieldEnums) { boolean modified = false; if (fieldEnums != null && !fieldEnums.contains(Field.PRIMARY_FILTERS)) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java index 23bca34..589255a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java @@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespaces; /** * This interface is for retrieving timeline information. @@ -152,4 +154,25 @@ TimelineEntity getEntity(String entityId, String entityType, EnumSet TimelineEvents getEntityTimelines(String entityType, SortedSet entityIds, Long limit, Long windowStart, Long windowEnd, Set eventTypes) throws IOException; + + /** + * This method retrieves the namespace information for a given ID. + * + * @return a {@link TimelineNamespace} object. + * @throws IOException + */ + TimelineNamespace getNamespace( + String namespaceId) throws IOException; + + /** + * This method retrieves all the namespaces that belong to a given owner. + * The namespaces are sorted according to the created time firstly and the + * modified time secondly in descending order. + * + * @param owner + * the namespace owner + * @return an {@link TimelineNamespaces} object. + * @throws IOException + */ + TimelineNamespaces getNamespaces(String owner) throws IOException; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java index a3e5aeb..76486e1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java @@ -18,13 +18,14 @@ package org.apache.hadoop.yarn.server.timeline; +import java.io.IOException; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; -import java.io.IOException; - /** * This interface is for storing timeline information. */ @@ -37,10 +38,21 @@ * individual put request objects will be reported in the response. * * @param data - * An {@link TimelineEntities} object. - * @return An {@link TimelinePutResponse} object. + * a {@link TimelineEntities} object. + * @return a {@link TimelinePutResponse} object. * @throws IOException */ TimelinePutResponse put(TimelineEntities data) throws IOException; + /** + * Store namespace information to the timeline store. If A namespace of the + * same ID already exists in the timeline store, it will be COMPLETELY updated + * with the given namespace. + * + * @param namespace + * a {@link TimelineNamespace} object + * @throws IOException + */ + void put(TimelineNamespace namespace) throws IOException; + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java index 10e62d2..4f6a99a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AdminACLsManager; import org.apache.hadoop.yarn.server.timeline.EntityIdentifier; @@ -81,6 +82,31 @@ public boolean checkAccess(UserGroupInformation callerUGI, return false; } + public boolean checkAccess(UserGroupInformation callerUGI, + TimelineNamespace namespace) throws YarnException, IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Verifying the access of " + + (callerUGI == null ? null : callerUGI.getShortUserName()) + + " on the timeline namespace " + namespace); + } + + if (!adminAclsManager.areACLsEnabled()) { + return true; + } + + String owner = namespace.getOwner(); + if (owner == null || owner.length() == 0) { + throw new YarnException("Owner information of the timeline namespace " + + namespace.getId() + " is corrupted."); + } + if (callerUGI != null + && (adminAclsManager.isAdmin(callerUGI) || + callerUGI.getShortUserName().equals(owner))) { + return true; + } + return false; + } + @Private @VisibleForTesting public AdminACLsManager diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java index c5e6d49..93f77ee 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.timeline.webapp; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; @@ -32,6 +33,7 @@ import javax.ws.rs.Consumes; import javax.ws.rs.GET; import javax.ws.rs.POST; +import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; @@ -40,6 +42,7 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; @@ -53,7 +56,10 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespaces; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.timeline.EntityIdentifier; import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper; import org.apache.hadoop.yarn.server.timeline.NameValuePair; @@ -259,6 +265,100 @@ public TimelinePutResponse postEntities( } } + /** + * Store the given namespace into the timeline store, and return the errors + * that happen during storing. + */ + @PUT + @Path("/namespace") + @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) + public Response putNamespace( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + TimelineNamespace namespace) { + init(res); + UserGroupInformation callerUGI = getUser(req); + if (callerUGI == null) { + String msg = "The owner of the posted timeline namespace is not set"; + LOG.error(msg); + throw new ForbiddenException(msg); + } + namespace.setOwner(callerUGI.getShortUserName()); + try { + timelineDataManager.putNamespace(namespace, callerUGI); + } catch (YarnException e) { + // The user doesn't have the access to override the existing namespace. + LOG.error(e.getMessage(), e); + throw new ForbiddenException(e); + } catch (IOException e) { + LOG.error("Error putting namespace", e); + throw new WebApplicationException(e, + Response.Status.INTERNAL_SERVER_ERROR); + } + return Response.status(Status.OK).build(); + } + + /** + * Return a single namespace of the given namespace Id. + */ + @GET + @Path("/namespace/{namespaceId}") + @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) + public TimelineNamespace getNamespace( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("namespaceId") String namespaceId) { + init(res); + namespaceId = parseStr(namespaceId); + if (namespaceId == null || namespaceId.length() == 0) { + throw new BadRequestException("Namespace ID is not specified."); + } + TimelineNamespace namespace = null; + try { + namespace = timelineDataManager.getNamspace( + parseStr(namespaceId), getUser(req)); + } catch (Exception e) { + LOG.error("Error getting namespace", e); + throw new WebApplicationException(e, + Response.Status.INTERNAL_SERVER_ERROR); + } + if (namespace == null) { + throw new NotFoundException("Timeline namespace [" + + namespaceId + "] is not found"); + } + return namespace; + } + + /** + * Return a list of namespaces of the given owner. + */ + @GET + @Path("/namespace") + @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) + public TimelineNamespaces getNamespaces( + @Context HttpServletRequest req, + @Context HttpServletResponse res, + @QueryParam("owner") String owner) { + init(res); + owner = parseStr(owner); + UserGroupInformation callerUGI = getUser(req); + if (owner == null || owner.length() == 0) { + if (callerUGI == null) { + throw new BadRequestException("Namespace owner is not specified."); + } else { + // By default it's going to list the caller's namespaces + owner = callerUGI.getShortUserName(); + } + } + try { + return timelineDataManager.getNamespaces(owner, callerUGI); + } catch (Exception e) { + LOG.error("Error getting namespaces", e); + throw new WebApplicationException(e, + Response.Status.INTERNAL_SERVER_ERROR); + } + } + private void init(HttpServletResponse response) { response.setContentType(null); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java index 0c6e082..b752bcc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java @@ -69,8 +69,9 @@ public void setup() throws Exception { store = new LeveldbTimelineStore(); store.init(config); store.start(); - loadTestData(); - loadVerificationData(); + loadTestEntityData(); + loadVerificationEntityData(); + loadTestNamespaceData(); } @After @@ -93,7 +94,7 @@ public void testGetSingleEntity() throws IOException { super.testGetSingleEntity(); ((LeveldbTimelineStore)store).clearStartTimeCache(); super.testGetSingleEntity(); - loadTestData(); + loadTestEntityData(); } @Test @@ -257,7 +258,7 @@ public void testFromTsWithDeletion() assertEquals(0, getEntities("type_2").size()); assertEquals(0, getEntitiesFromTsWithPrimaryFilter("type_1", userFilter, l).size()); - loadTestData(); + loadTestEntityData(); assertEquals(0, getEntitiesFromTs("type_1", l).size()); assertEquals(0, getEntitiesFromTs("type_2", l).size()); assertEquals(0, getEntitiesFromTsWithPrimaryFilter("type_1", userFilter, @@ -309,4 +310,14 @@ private void restartTimelineStore() throws IOException { store.start(); } + @Test + public void testGetNamespace() throws IOException { + super.testGetNamespace(); + } + + @Test + public void testGetNamespaces() throws IOException { + super.testGetNamespaces(); + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java index 1953442..22921f2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java @@ -34,8 +34,9 @@ public void setup() throws Exception { store = new MemoryTimelineStore(); store.init(new YarnConfiguration()); store.start(); - loadTestData(); - loadVerificationData(); + loadTestEntityData(); + loadVerificationEntityData(); + loadTestNamespaceData(); } @After @@ -82,4 +83,14 @@ public void testGetEvents() throws IOException { super.testGetEvents(); } + @Test + public void testGetNamespace() throws IOException { + super.testGetNamespace(); + } + + @Test + public void testGetNamespaces() throws IOException { + super.testGetNamespaces(); + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java index e8a6d83..0980adf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -38,11 +39,11 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; -import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespaces; +import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; -import org.apache.hadoop.yarn.server.timeline.NameValuePair; -import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; public class TimelineStoreTestUtils { @@ -88,9 +89,9 @@ protected long beforeTs; /** - * Load test data into the given store + * Load test entity data into the given store */ - protected void loadTestData() throws IOException { + protected void loadTestEntityData() throws IOException { beforeTs = System.currentTimeMillis()-1; TimelineEntities entities = new TimelineEntities(); Map> primaryFilters = @@ -184,9 +185,9 @@ protected void loadTestData() throws IOException { } /** - * Load verification data + * Load verification entity data */ - protected void loadVerificationData() throws Exception { + protected void loadVerificationEntityData() throws Exception { userFilter = new NameValuePair("user", "username"); numericFilter1 = new NameValuePair("appname", Integer.MAX_VALUE); numericFilter2 = new NameValuePair("long", (long)Integer.MAX_VALUE + 1l); @@ -263,6 +264,51 @@ protected void loadVerificationData() throws Exception { events2.add(ev4); } + private TimelineNamespace namespace1; + private TimelineNamespace namespace2; + private TimelineNamespace namespace3; + private long elapsedTime; + + protected void loadTestNamespaceData() throws IOException { + namespace1 = new TimelineNamespace(); + namespace1.setId("namespace_id_1"); + namespace1.setDescription("description_1"); + namespace1.setOwner("owner_1"); + namespace1.addReaders(Arrays.asList("reader_user_1", "reader_group_1")); + namespace1.addWriters(Arrays.asList("writer_user_1", "writer_group_1")); + store.put(namespace1); + + namespace2 = new TimelineNamespace(); + namespace2.setId("namespace_id_2"); + namespace2.setDescription("description_2"); + namespace2.setOwner("owner_2"); + namespace2.addReaders(Arrays.asList("reader_user_2", "reader_group_2")); + namespace2.addWriters(Arrays.asList("writer_user_2", "writer_group_2")); + store.put(namespace2); + + // Wait a second before updating the namespace information + elapsedTime = 1000; + try { + Thread.sleep(elapsedTime); + } catch (InterruptedException e) { + throw new IOException(e); + } + + namespace2.setDescription("description_3"); + namespace2.setOwner("owner_3"); + namespace2.setReaders(Arrays.asList("reader_user_3", "reader_group_3")); + namespace2.setWriters(Arrays.asList("writer_user_3", "writer_group_3")); + store.put(namespace2); + + namespace3 = new TimelineNamespace(); + namespace3.setId("namespace_id_4"); + namespace3.setDescription("description_4"); + namespace3.setOwner("owner_1"); + namespace3.addReaders(Arrays.asList("reader_user_4", "reader_group_4")); + namespace3.addWriters(Arrays.asList("writer_user_4", "writer_group_4")); + store.put(namespace3); + } + public void testGetSingleEntity() throws IOException { // test getting entity info verifyEntityInfo(null, null, null, null, null, null, @@ -519,7 +565,7 @@ public void testGetEntitiesWithFromTs() throws IOException { assertEquals(2, getEntitiesWithPrimaryFilter("type_1", userFilter).size()); // check insert time is not overwritten long beforeTs = this.beforeTs; - loadTestData(); + loadTestEntityData(); assertEquals(0, getEntitiesFromTs("type_1", beforeTs).size()); assertEquals(0, getEntitiesFromTs("type_2", beforeTs).size()); assertEquals(0, getEntitiesFromTsWithPrimaryFilter("type_1", userFilter, @@ -788,4 +834,45 @@ private static TimelineEvent createEvent(long timestamp, String type, Map 0); + assertTrue(actualNamespace1.getModifiedTime() > 0); + assertEquals( + actualNamespace1.getCreatedTime(), actualNamespace1.getModifiedTime()); + + TimelineNamespace actualNamespace2 = + store.getNamespace(namespace2.getId()); + verifyNamespaceInfo(namespace2, actualNamespace2); + assertEquals("namespace_id_2", actualNamespace2.getId()); + assertTrue(actualNamespace2.getCreatedTime() > 0); + assertTrue(actualNamespace2.getModifiedTime() > 0); + assertTrue( + actualNamespace2.getCreatedTime() < actualNamespace2.getModifiedTime()); + } + + public void testGetNamespaces() throws IOException { + TimelineNamespaces actualNamespaces = + store.getNamespaces("owner_1"); + assertEquals(2, actualNamespaces.getNamespaces().size()); + verifyNamespaceInfo(namespace3, actualNamespaces.getNamespaces().get(0)); + verifyNamespaceInfo(namespace1, actualNamespaces.getNamespaces().get(1)); + } + + private static void verifyNamespaceInfo( + TimelineNamespace expected, TimelineNamespace actual) { + assertEquals(expected.getId(), actual.getId()); + assertEquals(expected.getDescription(), actual.getDescription()); + assertEquals(expected.getOwner(), actual.getOwner()); + assertEquals(expected.getReaders().size(), actual.getReaders().size()); + for (int i = 0; i < expected.getReaders().size(); ++i) { + assertEquals(expected.getReaders().get(i), actual.getReaders().get(i)); + } + assertEquals(expected.getWriters().size(), actual.getWriters().size()); + for (int i = 0; i < expected.getWriters().size(); ++i) { + assertEquals(expected.getWriters().get(i), actual.getWriters().get(i)); + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java index 5825e7e..2e64db7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java @@ -21,17 +21,17 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.timeline.TimelineStore; -import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; import org.junit.Assert; import org.junit.Test; public class TestTimelineACLsManager { @Test - public void testYarnACLsNotEnabled() throws Exception { + public void testYarnACLsNotEnabledForEntity() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); TimelineACLsManager timelineACLsManager = @@ -47,7 +47,7 @@ public void testYarnACLsNotEnabled() throws Exception { } @Test - public void testYarnACLsEnabled() throws Exception { + public void testYarnACLsEnabledForEntity() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); @@ -72,7 +72,7 @@ public void testYarnACLsEnabled() throws Exception { } @Test - public void testCorruptedOwnerInfo() throws Exception { + public void testCorruptedOwnerInfoForEntity() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner"); @@ -89,4 +89,59 @@ public void testCorruptedOwnerInfo() throws Exception { } } + @Test + public void testYarnACLsNotEnabledForNamespace() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); + TimelineACLsManager timelineACLsManager = + new TimelineACLsManager(conf); + TimelineNamespace namespace = new TimelineNamespace(); + namespace.setOwner("owner"); + Assert.assertTrue( + "Always true when ACLs are not enabled", + timelineACLsManager.checkAccess( + UserGroupInformation.createRemoteUser("user"), namespace)); + } + + @Test + public void testYarnACLsEnabledForNamespace() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); + TimelineACLsManager timelineACLsManager = + new TimelineACLsManager(conf); + TimelineNamespace namespace = new TimelineNamespace(); + namespace.setOwner("owner"); + Assert.assertTrue( + "Owner should be allowed to access", + timelineACLsManager.checkAccess( + UserGroupInformation.createRemoteUser("owner"), namespace)); + Assert.assertFalse( + "Other shouldn't be allowed to access", + timelineACLsManager.checkAccess( + UserGroupInformation.createRemoteUser("other"), namespace)); + Assert.assertTrue( + "Admin should be allowed to access", + timelineACLsManager.checkAccess( + UserGroupInformation.createRemoteUser("admin"), namespace)); + } + + @Test + public void testCorruptedOwnerInfoForNamespace() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner"); + TimelineACLsManager timelineACLsManager = + new TimelineACLsManager(conf); + TimelineNamespace namespace = new TimelineNamespace(); + try { + timelineACLsManager.checkAccess( + UserGroupInformation.createRemoteUser("owner"), namespace); + Assert.fail("Exception is expected"); + } catch (YarnException e) { + Assert.assertTrue("It's not the exact expected exception", e.getMessage() + .contains("is corrupted.")); + } + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java index 549cfe1..8ce225c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java @@ -36,6 +36,7 @@ import javax.servlet.ServletContext; import javax.servlet.ServletException; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response.Status; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; @@ -44,6 +45,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace; +import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespaces; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -69,7 +72,6 @@ import com.sun.jersey.test.framework.JerseyTest; import com.sun.jersey.test.framework.WebAppDescriptor; - public class TestTimelineWebServices extends JerseyTest { private static TimelineStore store; @@ -85,7 +87,7 @@ protected void configureServlets() { bind(YarnJacksonJaxbJsonProvider.class); bind(TimelineWebServices.class); bind(GenericExceptionHandler.class); - try{ + try { store = mockTimelineStore(); } catch (Exception e) { Assert.fail(); @@ -100,7 +102,8 @@ protected void configureServlets() { new TimelineDataManager(store, timelineACLsManager); bind(TimelineDataManager.class).toInstance(timelineDataManager); serve("/*").with(GuiceContainer.class); - TimelineAuthenticationFilter taFilter = new TimelineAuthenticationFilter(); + TimelineAuthenticationFilter taFilter = + new TimelineAuthenticationFilter(); FilterConfig filterConfig = mock(FilterConfig.class); when(filterConfig.getInitParameter(AuthenticationFilter.CONFIG_PREFIX)) .thenReturn(null); @@ -159,7 +162,8 @@ public TestTimelineWebServices() { .filterClass(com.google.inject.servlet.GuiceFilter.class) .contextPath("jersey-guice-filter") .servletPath("/") - .clientConfig(new DefaultClientConfig(YarnJacksonJaxbJsonProvider.class)) + .clientConfig( + new DefaultClientConfig(YarnJacksonJaxbJsonProvider.class)) .build()); } @@ -277,7 +281,7 @@ public void testPrimaryFilterLong() { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") .path("type_1").queryParam("primaryFilter", - "long:" + Long.toString((long)Integer.MAX_VALUE + 1l)) + "long:" + Long.toString((long) Integer.MAX_VALUE + 1l)) .accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); @@ -406,7 +410,8 @@ public void testPostEntitiesWithPrimaryFilter() throws Exception { TimelineEntities entities = new TimelineEntities(); TimelineEntity entity = new TimelineEntity(); Map> filters = new HashMap>(); - filters.put(TimelineStore.SystemFilter.ENTITY_OWNER.toString(), new HashSet()); + filters.put(TimelineStore.SystemFilter.ENTITY_OWNER.toString(), + new HashSet()); entity.setPrimaryFilters(filters); entity.setEntityId("test id 6"); entity.setEntityType("test type 6"); @@ -418,13 +423,15 @@ public void testPostEntitiesWithPrimaryFilter() throws Exception { .accept(MediaType.APPLICATION_JSON) .type(MediaType.APPLICATION_JSON) .post(ClientResponse.class, entities); - TimelinePutResponse putResposne = response.getEntity(TimelinePutResponse.class); + TimelinePutResponse putResposne = + response.getEntity(TimelinePutResponse.class); Assert.assertEquals(1, putResposne.getErrors().size()); List errors = putResposne.getErrors(); - Assert.assertEquals(TimelinePutResponse.TimelinePutError.SYSTEM_FILTER_CONFLICT, - errors.get(0).getErrorCode()); + Assert.assertEquals( + TimelinePutResponse.TimelinePutError.SYSTEM_FILTER_CONFLICT, + errors.get(0).getErrorCode()); } - + @Test public void testPostEntities() throws Exception { TimelineEntities entities = new TimelineEntities(); @@ -449,7 +456,8 @@ public void testPostEntities() throws Exception { .type(MediaType.APPLICATION_JSON) .post(ClientResponse.class, entities); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); - TimelinePutResponse putResposne = response.getEntity(TimelinePutResponse.class); + TimelinePutResponse putResposne = + response.getEntity(TimelinePutResponse.class); Assert.assertNotNull(putResposne); Assert.assertEquals(0, putResposne.getErrors().size()); // verify the entity exists in the store @@ -482,7 +490,8 @@ public void testPostEntitiesWithYarnACLsEnabled() throws Exception { .type(MediaType.APPLICATION_JSON) .post(ClientResponse.class, entities); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); - TimelinePutResponse putResponse = response.getEntity(TimelinePutResponse.class); + TimelinePutResponse putResponse = + response.getEntity(TimelinePutResponse.class); Assert.assertNotNull(putResponse); Assert.assertEquals(0, putResponse.getErrors().size()); @@ -668,4 +677,202 @@ public void testGetEventsWithYarnACLsEnabled() { } } + @Test + public void testGetNamespace() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("timeline") + .path("namespace").path("namespace_id_1") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + TimelineNamespace namespace = response.getEntity(TimelineNamespace.class); + verifyNamespace(namespace, "namespace_id_1", true); + } + + @Test + public void testGetNamespaceYarnACLsEnabled() { + AdminACLsManager oldAdminACLsManager = + timelineACLsManager.setAdminACLsManager(adminACLsManager); + try { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("timeline") + .path("namespace").path("namespace_id_1") + .queryParam("user.name", "owner_1") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + TimelineNamespace namespace = response.getEntity(TimelineNamespace.class); + verifyNamespace(namespace, "namespace_id_1", true); + + response = r.path("ws").path("v1").path("timeline") + .path("namespace").path("namespace_id_1") + .queryParam("user.name", "tester") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + namespace = response.getEntity(TimelineNamespace.class); + verifyNamespace(namespace, "namespace_id_1", false); + } finally { + timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); + } + } + + @Test + public void testGetNamespaces() throws Exception { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("timeline") + .path("namespace") + .queryParam("owner", "owner_1") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + TimelineNamespaces namespaces = response.getEntity(TimelineNamespaces.class); + Assert.assertEquals(2, namespaces.getNamespaces().size()); + for (int i = 0; i < namespaces.getNamespaces().size(); ++i) { + verifyNamespace(namespaces.getNamespaces().get(i), + i == 0 ? "namespace_id_4" : "namespace_id_1", true); + } + } + + @Test + public void testGetNamespacesYarnACLsEnabled() throws Exception { + AdminACLsManager oldAdminACLsManager = + timelineACLsManager.setAdminACLsManager(adminACLsManager); + try { + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("timeline") + .path("namespace") + .queryParam("user.name", "owner_1") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + TimelineNamespaces namespaces = response.getEntity(TimelineNamespaces.class); + Assert.assertEquals(2, namespaces.getNamespaces().size()); + for (int i = 0; i < namespaces.getNamespaces().size(); ++i) { + verifyNamespace(namespaces.getNamespaces().get(i), + i == 0 ? "namespace_id_4" : "namespace_id_1", true); + } + + response = r.path("ws").path("v1").path("timeline") + .path("namespace") + .queryParam("owner", "owner_1") + .queryParam("user.name", "tester") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + namespaces = response.getEntity(TimelineNamespaces.class); + Assert.assertEquals(2, namespaces.getNamespaces().size()); + for (int i = 0; i < namespaces.getNamespaces().size(); ++i) { + verifyNamespace(namespaces.getNamespaces().get(i), + i == 0 ? "namespace_id_4" : "namespace_id_1", false); + } + } finally { + timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); + } + } + + @Test + public void testPutNamespace() throws Exception { + TimelineNamespace namespace = new TimelineNamespace(); + namespace.setId("test_namespace_id"); + WebResource r = resource(); + // No owner, will be rejected + ClientResponse response = r.path("ws").path("v1") + .path("timeline").path("namespace") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .put(ClientResponse.class, namespace); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + assertEquals(ClientResponse.Status.FORBIDDEN, + response.getClientResponseStatus()); + + response = r.path("ws").path("v1") + .path("timeline").path("namespace") + .queryParam("user.name", "tester") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .put(ClientResponse.class, namespace); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + + // Verify the namespace exists + response = r.path("ws").path("v1").path("timeline") + .path("namespace").path("test_namespace_id") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + namespace = response.getEntity(TimelineNamespace.class); + Assert.assertNotNull(namespace); + Assert.assertEquals("test_namespace_id", namespace.getId()); + Assert.assertEquals("tester", namespace.getOwner()); + Assert.assertEquals(null, namespace.getDescription()); + + // Update the namespace + namespace.setDescription("test_description"); + response = r.path("ws").path("v1") + .path("timeline").path("namespace") + .queryParam("user.name", "tester") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .put(ClientResponse.class, namespace); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + + // Verify the namespace is updated + response = r.path("ws").path("v1").path("timeline") + .path("namespace").path("test_namespace_id") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + namespace = response.getEntity(TimelineNamespace.class); + Assert.assertNotNull(namespace); + Assert.assertEquals("test_namespace_id", namespace.getId()); + Assert.assertEquals("test_description", namespace.getDescription()); + } + + @Test + public void testPutNamespaceYarnACLsEnabled() throws Exception { + AdminACLsManager oldAdminACLsManager = + timelineACLsManager.setAdminACLsManager(adminACLsManager); + try { + TimelineNamespace namespace = new TimelineNamespace(); + namespace.setId("test_namespace_id_acl"); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1") + .path("timeline").path("namespace") + .queryParam("user.name", "tester") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .put(ClientResponse.class, namespace); + assertEquals(Status.OK.getStatusCode(), response.getStatus()); + + // Update the namespace by another user + response = r.path("ws").path("v1") + .path("timeline").path("namespace") + .queryParam("user.name", "other") + .accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .put(ClientResponse.class, namespace); + assertEquals(Status.FORBIDDEN.getStatusCode(), response.getStatus()); + } finally { + timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); + } + } + + private static void verifyNamespace(TimelineNamespace namespace, + String namespaceId, boolean hasAccess) { + Assert.assertNotNull(namespace); + Assert.assertEquals(namespaceId, namespace.getId()); + // The specific values have been verified in TestMemoryTimelineStore + Assert.assertTrue(hasAccess && namespace.getDescription() != null || + !hasAccess && namespace.getDescription() == null); + Assert.assertTrue(hasAccess && namespace.getOwner() != null || + !hasAccess && namespace.getOwner() == null); + Assert.assertTrue(hasAccess && namespace.getReaders().size() != 0 || + !hasAccess && namespace.getReaders().size() == 0); + Assert.assertTrue(hasAccess && namespace.getWriters().size() != 0 || + !hasAccess && namespace.getWriters().size() == 0); + Assert.assertTrue(hasAccess && namespace.getCreatedTime() != null || + !hasAccess && namespace.getCreatedTime() == null); + Assert.assertTrue(hasAccess && namespace.getModifiedTime() != null || + !hasAccess && namespace.getModifiedTime() == null); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java index 81f87fb..7c1fe16 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java @@ -24,7 +24,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; @@ -124,8 +123,8 @@ public void testPutEntities() throws Exception { private ClientResponse resp; @Override - public ClientResponse doPostingEntities(TimelineEntities entities) { - resp = super.doPostingEntities(entities); + public ClientResponse doPostingObject(Object obj, String path) { + resp = super.doPostingObject(obj, path); return resp; }