diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineNamespace.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineNamespace.java
new file mode 100644
index 0000000..33149be
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineNamespace.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.timeline;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ *
+ * This class contains the information about a timeline namespace, which is used
+ * to a user to host a number of timeline entities, isolating them from others'.
+ * The user can also define the reader and writer users/groups for the the
+ * namespace, which is used to control the access to its entities.
+ *
+ *
+ *
+ * The reader and writer users/groups pattern that the user can supply is the
+ * same as what AccessControlList takes.
+ *
+ *
+ */
+@XmlRootElement(name = "namespace")
+@XmlAccessorType(XmlAccessType.NONE)
+@Public
+@Unstable
+public class TimelineNamespace {
+
+ private String id;
+ private String description;
+ private String owner;
+ private String readers;
+ private String writers;
+ private Long createdTime;
+ private Long modifiedTime;
+
+ public TimelineNamespace() {
+ }
+
+ /**
+ * Get the namespace ID
+ *
+ * @return the namespace ID
+ */
+ @XmlElement(name = "id")
+ public String getId() {
+ return id;
+ }
+
+ /**
+ * Set the namespace ID
+ *
+ * @param id the namespace ID
+ */
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
+ * Get the namespace description
+ *
+ * @return the namespace description
+ */
+ @XmlElement(name = "description")
+ public String getDescription() {
+ return description;
+ }
+
+ /**
+ * Set the namespace description
+ *
+ * @param description the namespace description
+ */
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ /**
+ * Get the namespace owner
+ *
+ * @return the namespace owner
+ */
+ @XmlElement(name = "owner")
+ public String getOwner() {
+ return owner;
+ }
+
+ /**
+ * Set the namespace owner. The user doesn't need to set it, which will
+ * automatically set to the user who puts the namespace.
+ *
+ * @param owner the namespace owner
+ */
+ public void setOwner(String owner) {
+ this.owner = owner;
+ }
+
+ /**
+ * Get the reader (and/or reader group) list string
+ *
+ * @return the reader (and/or reader group) list string
+ */
+ @XmlElement(name = "readers")
+ public String getReaders() {
+ return readers;
+ }
+
+ /**
+ * Set the reader (and/or reader group) list string
+ *
+ * @param readers the reader (and/or reader group) list string
+ */
+ public void setReaders(String readers) {
+ this.readers = readers;
+ }
+
+ /**
+ * Get the writer (and/or writer group) list string
+ *
+ * @return the writer (and/or writer group) list string
+ */
+ @XmlElement(name = "writers")
+ public String getWriters() {
+ return writers;
+ }
+
+ /**
+ * Set the writer (and/or writer group) list string
+ *
+ * @param writers the writer (and/or writer group) list string
+ */
+ public void setWriters(String writers) {
+ this.writers = writers;
+ }
+
+ /**
+ * Get the created time of the namespace
+ *
+ * @return the created time of the namespace
+ */
+ @XmlElement(name = "createdtime")
+ public Long getCreatedTime() {
+ return createdTime;
+ }
+
+ /**
+ * Set the created time of the namespace
+ *
+ * @param createdTime the created time of the namespace
+ */
+ public void setCreatedTime(Long createdTime) {
+ this.createdTime = createdTime;
+ }
+
+ /**
+ * Get the modified time of the namespace
+ *
+ * @return the modified time of the namespace
+ */
+ @XmlElement(name = "modifiedtime")
+ public Long getModifiedTime() {
+ return modifiedTime;
+ }
+
+ /**
+ * Set the modified time of the namespace
+ *
+ * @param modifiedTime the modified time of the namespace
+ */
+ public void setModifiedTime(Long modifiedTime) {
+ this.modifiedTime = modifiedTime;
+ }
+
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineNamespaces.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineNamespaces.java
new file mode 100644
index 0000000..f8473f8
--- /dev/null
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineNamespaces.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.timeline;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * The class that hosts a list of timeline namespaces.
+ */
+@XmlRootElement(name = "namespaces")
+@XmlAccessorType(XmlAccessType.NONE)
+@Public
+@Unstable
+public class TimelineNamespaces {
+
+ private List namespaces =
+ new ArrayList();
+
+ public TimelineNamespaces() {
+ }
+
+ /**
+ * Get a list of namespaces
+ *
+ * @return a list of namespaces
+ */
+ @XmlElement(name = "namespaces")
+ public List getNamespaces() {
+ return namespaces;
+ }
+
+ /**
+ * Add a single namespace into the existing namespace list
+ *
+ * @param namespace
+ * a single namespace
+ */
+ public void addNamespace(TimelineNamespace namespace) {
+ namespaces.add(namespace);
+ }
+
+ /**
+ * All a list of namespaces into the existing namespace list
+ *
+ * @param namespaces
+ * a list of namespaces
+ */
+ public void addNamespaces(List namespaces) {
+ this.namespaces.addAll(namespaces);
+ }
+
+ /**
+ * Set the namespace list to the given list of namespaces
+ *
+ * @param namespaces
+ * a list of namespaces
+ */
+ public void setNamespaces(List namespaces) {
+ this.namespaces = namespaces;
+ }
+
+}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
index de1d3e2..fa86be3 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -69,6 +70,22 @@ public abstract TimelinePutResponse putEntities(
/**
*
+ * Send the information of a namespace to the timeline server. It is a
+ * blocking API. The method will not return until it gets the response from
+ * the timeline server.
+ *
+ *
+ * @param namespace
+ * an {@link TimelineNamespace} object
+ * @throws IOException
+ * @throws YarnException
+ */
+ @Public
+ public abstract void putNamespace(
+ TimelineNamespace namespace) throws IOException, YarnException;
+
+ /**
+ *
* Get a delegation token so as to be able to talk to the timeline server in a
* secure way.
*
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index f383a8a..dbfed54 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -52,6 +52,7 @@
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -150,9 +151,27 @@ public TimelinePutResponse putEntities(
}
TimelineEntities entitiesContainer = new TimelineEntities();
entitiesContainer.addEntities(Arrays.asList(entities));
+ ClientResponse resp = doPosting(entitiesContainer, null);
+ return resp.getEntity(TimelinePutResponse.class);
+ }
+
+
+ @Override
+ public void putNamespace(TimelineNamespace namespace) throws IOException,
+ YarnException {
+ if (!isEnabled) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Nothing will be put because timeline service is not enabled");
+ }
+ return;
+ }
+ doPosting(namespace, "namespace");
+ }
+
+ private ClientResponse doPosting(Object obj, String path) throws IOException, YarnException {
ClientResponse resp;
try {
- resp = doPostingEntities(entitiesContainer);
+ resp = doPostingObject(obj, path);
} catch (RuntimeException re) {
// runtime exception is expected if the client cannot connect the server
String msg =
@@ -172,7 +191,7 @@ public TimelinePutResponse putEntities(
}
throw new YarnException(msg);
}
- return resp.getEntity(TimelinePutResponse.class);
+ return resp;
}
@Override
@@ -184,11 +203,14 @@ public TimelinePutResponse putEntities(
@Private
@VisibleForTesting
- public ClientResponse doPostingEntities(TimelineEntities entities) {
+ public ClientResponse doPostingObject(Object object, String path) {
WebResource webResource = client.resource(resURI);
+ if (path != null) {
+ webResource.path(path);
+ }
return webResource.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
- .post(ClientResponse.class, entities);
+ .post(ClientResponse.class, object);
}
private static class PseudoAuthenticatedURLConnectionFactory
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java
index 7813b5d..8d89b73 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java
@@ -161,4 +161,42 @@ public void testTimelinePutErrors() throws Exception {
Assert.assertEquals(error2.getErrorCode(), e.getErrorCode());
}
+ @Test
+ public void testTimelineNamespace() throws Exception {
+ TimelineNamespaces namespaces = new TimelineNamespaces();
+
+ TimelineNamespace namespace = null;
+ for (int i = 0; i < 2; ++i) {
+ namespace = new TimelineNamespace();
+ namespace.setId("test id " + (i + 1));
+ namespace.setDescription("test description " + (i + 1));
+ namespace.setOwner("test owner " + (i + 1));
+ namespace.setReaders("test_reader_user_" + (i + 1) +
+ " test_reader_group+" + (i + 1));
+ namespace.setWriters("test_writer_user_" + (i + 1) +
+ " test_writer_group+" + (i + 1));
+ namespace.setCreatedTime(0L);
+ namespace.setModifiedTime(1L);
+ LOG.info("Namespace in JSON:");
+ LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(namespace, true));
+ namespaces.addNamespace(namespace);
+ }
+
+ Assert.assertEquals(2, namespaces.getNamespaces().size());
+
+ for (int i = 0; i < namespaces.getNamespaces().size(); ++i) {
+ namespace = namespaces.getNamespaces().get(i);
+ Assert.assertEquals("test id " + (i + 1), namespace.getId());
+ Assert.assertEquals("test description " + (i + 1),
+ namespace.getDescription());
+ Assert.assertEquals("test owner " + (i + 1), namespace.getOwner());
+ Assert.assertEquals("test_reader_user_" + (i + 1) +
+ " test_reader_group+" + (i + 1), namespace.getReaders());
+ Assert.assertEquals("test_writer_user_" + (i + 1) +
+ " test_writer_group+" + (i + 1), namespace.getWriters());
+ Assert.assertEquals(new Long(0L), namespace.getCreatedTime());
+ Assert.assertEquals(new Long(1L), namespace.getModifiedTime());
+ }
+ }
+
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
index 3c5272a..e95187d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
@@ -27,16 +27,16 @@
import java.net.ConnectException;
-import org.junit.Assert;
-
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -63,7 +63,7 @@ public void tearDown() {
@Test
public void testPostEntities() throws Exception {
- mockClientResponse(client, ClientResponse.Status.OK, false, false);
+ mockEntityClientResponse(client, ClientResponse.Status.OK, false, false);
try {
TimelinePutResponse response = client.putEntities(generateEntity());
Assert.assertEquals(0, response.getErrors().size());
@@ -74,7 +74,7 @@ public void testPostEntities() throws Exception {
@Test
public void testPostEntitiesWithError() throws Exception {
- mockClientResponse(client, ClientResponse.Status.OK, true, false);
+ mockEntityClientResponse(client, ClientResponse.Status.OK, true, false);
try {
TimelinePutResponse response = client.putEntities(generateEntity());
Assert.assertEquals(1, response.getErrors().size());
@@ -91,7 +91,7 @@ public void testPostEntitiesWithError() throws Exception {
@Test
public void testPostEntitiesNoResponse() throws Exception {
- mockClientResponse(
+ mockEntityClientResponse(
client, ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false);
try {
client.putEntities(generateEntity());
@@ -104,7 +104,7 @@ public void testPostEntitiesNoResponse() throws Exception {
@Test
public void testPostEntitiesConnectionRefused() throws Exception {
- mockClientResponse(client, null, false, true);
+ mockEntityClientResponse(client, null, false, true);
try {
client.putEntities(generateEntity());
Assert.fail("RuntimeException is expected");
@@ -118,7 +118,7 @@ public void testPostEntitiesTimelineServiceNotEnabled() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
TimelineClientImpl client = createTimelineClient(conf);
- mockClientResponse(
+ mockEntityClientResponse(
client, ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false);
try {
TimelinePutResponse response = client.putEntities(generateEntity());
@@ -137,7 +137,7 @@ public void testPostEntitiesTimelineServiceDefaultNotEnabled()
// Make sure default value is pickup up
conf.unset(YarnConfiguration.TIMELINE_SERVICE_ENABLED);
TimelineClientImpl client = createTimelineClient(conf);
- mockClientResponse(client, ClientResponse.Status.INTERNAL_SERVER_ERROR,
+ mockEntityClientResponse(client, ClientResponse.Status.INTERNAL_SERVER_ERROR,
false, false);
try {
TimelinePutResponse response = client.putEntities(generateEntity());
@@ -148,16 +148,50 @@ public void testPostEntitiesTimelineServiceDefaultNotEnabled()
}
}
- private static ClientResponse mockClientResponse(TimelineClientImpl client,
- ClientResponse.Status status, boolean hasError, boolean hasRuntimeError) {
+ @Test
+ public void testPutNamespace() throws Exception {
+ mockNamespaceClientResponse(client, ClientResponse.Status.OK, false);
+ try {
+ client.putNamespace(generateNamespace());
+ } catch (YarnException e) {
+ Assert.fail("Exception is not expected");
+ }
+ }
+
+ @Test
+ public void testPutNamespaceNoResponse() throws Exception {
+ mockNamespaceClientResponse(client, ClientResponse.Status.FORBIDDEN, false);
+ try {
+ client.putNamespace(generateNamespace());
+ Assert.fail("Exception is expected");
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Failed to get the response from the timeline server."));
+ }
+ }
+
+ @Test
+ public void testPutNamespaceConnectionRefused() throws Exception {
+ mockNamespaceClientResponse(client, null, true);
+ try {
+ client.putNamespace(generateNamespace());
+ Assert.fail("RuntimeException is expected");
+ } catch (RuntimeException re) {
+ Assert.assertTrue(re instanceof ClientHandlerException);
+ }
+ }
+
+ private static ClientResponse mockEntityClientResponse(
+ TimelineClientImpl client, ClientResponse.Status status,
+ boolean hasError, boolean hasRuntimeError) {
ClientResponse response = mock(ClientResponse.class);
if (hasRuntimeError) {
doThrow(new ClientHandlerException(new ConnectException())).when(client)
- .doPostingEntities(any(TimelineEntities.class));
+ .doPostingObject(any(TimelineEntities.class), any(String.class));
return response;
}
doReturn(response).when(client)
- .doPostingEntities(any(TimelineEntities.class));
+ .doPostingObject(any(TimelineEntities.class), any(String.class));
when(response.getClientResponseStatus()).thenReturn(status);
TimelinePutResponse.TimelinePutError error =
new TimelinePutResponse.TimelinePutError();
@@ -172,6 +206,21 @@ private static ClientResponse mockClientResponse(TimelineClientImpl client,
return response;
}
+ private static ClientResponse mockNamespaceClientResponse(
+ TimelineClientImpl client, ClientResponse.Status status,
+ boolean hasRuntimeError) {
+ ClientResponse response = mock(ClientResponse.class);
+ if (hasRuntimeError) {
+ doThrow(new ClientHandlerException(new ConnectException())).when(client)
+ .doPostingObject(any(TimelineNamespace.class), any(String.class));
+ return response;
+ }
+ doReturn(response).when(client)
+ .doPostingObject(any(TimelineNamespace.class), any(String.class));
+ when(response.getClientResponseStatus()).thenReturn(status);
+ return response;
+ }
+
private static TimelineEntity generateEntity() {
TimelineEntity entity = new TimelineEntity();
entity.setEntityId("entity id");
@@ -194,6 +243,18 @@ private static TimelineEntity generateEntity() {
return entity;
}
+ public static TimelineNamespace generateNamespace() {
+ TimelineNamespace namespace = new TimelineNamespace();
+ namespace.setId("namesapce id");
+ namespace.setDescription("namespace description");
+ namespace.setOwner("namespace owner");
+ namespace.setReaders("namespace_reader");
+ namespace.setWriters("namespace_writer");
+ namespace.setCreatedTime(0L);
+ namespace.setModifiedTime(1L);
+ return namespace;
+ }
+
private static TimelineClientImpl createTimelineClient(
YarnConfiguration conf) {
TimelineClientImpl client =
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
index b0feac1..8d4fdbd 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
@@ -58,6 +58,8 @@
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespaces;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -145,6 +147,14 @@
private static final byte[] INVISIBLE_REVERSE_RELATED_ENTITIES_COLUMN =
"z".getBytes();
+ private static final byte[] NAMESPACE_ENTRY_PREFIX = "n".getBytes();
+ private static final byte[] OWNER_LOOKUP_PREFIX = "o".getBytes();
+ private static final byte[] DESCRIPTION_COLUMN = "d".getBytes();
+ private static final byte[] OWNER_COLUMN = "o".getBytes();
+ private static final byte[] READER_COLUMN = "r".getBytes();
+ private static final byte[] WRITER_COLUMN = "w".getBytes();
+ private static final byte[] TIMESTAMP_COLUMN = "t".getBytes();
+
private static final byte[] EMPTY_BYTES = new byte[0];
private static final String TIMELINE_STORE_VERSION_KEY = "timeline-store-version";
@@ -1558,5 +1568,204 @@ private void checkVersion() throws IOException {
throw new IOException(incompatibleMessage);
}
}
-
+
+ //TODO: make data retention work with the namespace data as well
+ @Override
+ public void put(TimelineNamespace namespace) throws IOException {
+ WriteBatch writeBatch = null;
+ try {
+ writeBatch = db.createWriteBatch();
+ if (namespace.getId() == null || namespace.getId().length() == 0) {
+ throw new IllegalArgumentException("Namespace doesn't have an ID");
+ }
+ if (namespace.getOwner() == null || namespace.getOwner().length() == 0) {
+ throw new IllegalArgumentException("Namespace doesn't have an owner.");
+ }
+
+ // Write description
+ byte[] namespaceEntryKey = createNamespaceEntryKey(
+ namespace.getId(), DESCRIPTION_COLUMN);
+ byte[] ownerLookupEntryKey = createOwnerLookupKey(
+ namespace.getOwner(), namespace.getId(), DESCRIPTION_COLUMN);
+ if (namespace.getDescription() != null) {
+ writeBatch.put(namespaceEntryKey, namespace.getDescription().getBytes());
+ writeBatch.put(ownerLookupEntryKey, namespace.getDescription().getBytes());
+ } else {
+ writeBatch.put(namespaceEntryKey, EMPTY_BYTES);
+ writeBatch.put(ownerLookupEntryKey, EMPTY_BYTES);
+ }
+
+ // Write owner
+ namespaceEntryKey = createNamespaceEntryKey(namespace.getId(), OWNER_COLUMN);
+ ownerLookupEntryKey = createOwnerLookupKey(
+ namespace.getOwner(), namespace.getId(), OWNER_COLUMN);
+ // Null check for owner is done before
+ writeBatch.put(namespaceEntryKey, namespace.getOwner().getBytes());
+ writeBatch.put(ownerLookupEntryKey, namespace.getOwner().getBytes());
+
+ // Write readers
+ namespaceEntryKey = createNamespaceEntryKey(namespace.getId(), READER_COLUMN);
+ ownerLookupEntryKey = createOwnerLookupKey(
+ namespace.getOwner(), namespace.getId(), READER_COLUMN);
+ if (namespace.getReaders() != null && namespace.getReaders().length() > 0) {
+ writeBatch.put(namespaceEntryKey, namespace.getReaders().getBytes());
+ writeBatch.put(ownerLookupEntryKey, namespace.getReaders().getBytes());
+ } else {
+ writeBatch.put(namespaceEntryKey, EMPTY_BYTES);
+ writeBatch.put(ownerLookupEntryKey, EMPTY_BYTES);
+ }
+
+ // Write writers
+ namespaceEntryKey = createNamespaceEntryKey(namespace.getId(), WRITER_COLUMN);
+ ownerLookupEntryKey = createOwnerLookupKey(
+ namespace.getOwner(), namespace.getId(), WRITER_COLUMN);
+ if (namespace.getWriters() != null && namespace.getWriters().length() > 0) {
+ writeBatch.put(namespaceEntryKey, namespace.getWriters().getBytes());
+ writeBatch.put(ownerLookupEntryKey, namespace.getWriters().getBytes());
+ } else {
+ writeBatch.put(namespaceEntryKey, EMPTY_BYTES);
+ writeBatch.put(ownerLookupEntryKey, EMPTY_BYTES);
+ }
+
+ // Write creation time and modification time
+ namespaceEntryKey = createNamespaceEntryKey(namespace.getId(), TIMESTAMP_COLUMN);
+ ownerLookupEntryKey = createOwnerLookupKey(
+ namespace.getOwner(), namespace.getId(), TIMESTAMP_COLUMN);
+ long currentTimestamp = System.currentTimeMillis();
+ byte[] timestamps = db.get(namespaceEntryKey);
+ if (timestamps == null) {
+ timestamps = new byte[16];
+ writeReverseOrderedLong(currentTimestamp, timestamps, 0);
+ writeReverseOrderedLong(currentTimestamp, timestamps, 8);
+ } else {
+ writeReverseOrderedLong(currentTimestamp, timestamps, 8);
+ }
+ writeBatch.put(namespaceEntryKey, timestamps);
+ writeBatch.put(ownerLookupEntryKey, timestamps);
+ db.write(writeBatch);
+ } finally {
+ IOUtils.cleanup(LOG, writeBatch);
+ }
+ }
+
+ /**
+ * Creates a namespace entity key with column name suffix,
+ * of the form NAMESPACE_ENTRY_PREFIX + namespace id + column name.
+ */
+ private static byte[] createNamespaceEntryKey(String namespaceId,
+ byte[] columnName) throws IOException {
+ return KeyBuilder.newInstance().add(NAMESPACE_ENTRY_PREFIX)
+ .add(namespaceId).add(columnName).getBytes();
+ }
+
+ /**
+ * Creates an owner lookup key with column name suffix,
+ * of the form OWNER_LOOKUP_PREFIX + owner + namespace id + column name.
+ */
+ private static byte[] createOwnerLookupKey(
+ String owner, String namespaceId, byte[] columnName) throws IOException {
+ return KeyBuilder.newInstance().add(OWNER_LOOKUP_PREFIX)
+ .add(owner).add(namespaceId).add(columnName).getBytes();
+ }
+
+ @Override
+ public TimelineNamespace getNamespace(String namespaceId)
+ throws IOException {
+ DBIterator iterator = null;
+ try {
+ byte[] prefix = KeyBuilder.newInstance()
+ .add(NAMESPACE_ENTRY_PREFIX).add(namespaceId).getBytesForLookup();
+ iterator = db.iterator();
+ iterator.seek(prefix);
+ return getTimelineNamespace(iterator, namespaceId, prefix);
+ } finally {
+ IOUtils.cleanup(LOG, iterator);
+ }
+ }
+
+ @Override
+ public TimelineNamespaces getNamespaces(String owner)
+ throws IOException {
+ DBIterator iterator = null;
+ try {
+ byte[] prefix = KeyBuilder.newInstance()
+ .add(OWNER_LOOKUP_PREFIX).add(owner).getBytesForLookup();
+ List namespaces = new ArrayList();
+ for (iterator = db.iterator(), iterator.seek(prefix);
+ iterator.hasNext();) {
+ byte[] key = iterator.peekNext().getKey();
+ if (!prefixMatches(prefix, prefix.length, key)) {
+ break;
+ }
+ // Iterator to parse the rows of an individual namespace
+ KeyParser kp = new KeyParser(key, prefix.length);
+ String namespaceId = kp.getNextString();
+ byte[] prefixExt = KeyBuilder.newInstance().add(OWNER_LOOKUP_PREFIX)
+ .add(owner).add(namespaceId).getBytesForLookup();
+ TimelineNamespace namespaceToReturn =
+ getTimelineNamespace(iterator, namespaceId, prefixExt);
+ if (namespaceToReturn != null) {
+ namespaces.add(namespaceToReturn);
+ }
+ }
+ // Sort the namespaces to return
+ Collections.sort(namespaces, new Comparator() {
+ @Override
+ public int compare(
+ TimelineNamespace namespace1, TimelineNamespace namespace2) {
+ int result = namespace2.getCreatedTime().compareTo(
+ namespace1.getCreatedTime());
+ if (result == 0) {
+ return namespace2.getModifiedTime().compareTo(
+ namespace1.getModifiedTime());
+ } else {
+ return result;
+ }
+ }
+ });
+ TimelineNamespaces namespacesToReturn = new TimelineNamespaces();
+ namespacesToReturn.addNamespaces(namespaces);
+ return namespacesToReturn;
+ } finally {
+ IOUtils.cleanup(LOG, iterator);
+ }
+ }
+
+ private static TimelineNamespace getTimelineNamespace(
+ DBIterator iterator, String namespaceId, byte[] prefix) throws IOException {
+ // Iterate over all the rows whose key starts with prefix to retrieve the
+ // namespace information.
+ TimelineNamespace namespace = new TimelineNamespace();
+ namespace.setId(namespaceId);
+ boolean noRows = true;
+ for (; iterator.hasNext(); iterator.next()) {
+ byte[] key = iterator.peekNext().getKey();
+ if (!prefixMatches(prefix, prefix.length, key)) {
+ break;
+ }
+ if (noRows) {
+ noRows = false;
+ }
+ byte[] value = iterator.peekNext().getValue();
+ if (value != null && value.length > 0) {
+ if (key[prefix.length] == DESCRIPTION_COLUMN[0]) {
+ namespace.setDescription(new String(value));
+ } else if (key[prefix.length] == OWNER_COLUMN[0]) {
+ namespace.setOwner(new String(value));
+ } else if (key[prefix.length] == READER_COLUMN[0]) {
+ namespace.setReaders(new String(value));
+ } else if (key[prefix.length] == WRITER_COLUMN[0]) {
+ namespace.setWriters(new String(value));
+ } else if (key[prefix.length] == TIMESTAMP_COLUMN[0]) {
+ namespace.setCreatedTime(readReverseOrderedLong(value, 0));
+ namespace.setModifiedTime(readReverseOrderedLong(value, 8));
+ }
+ }
+ }
+ if (noRows) {
+ return null;
+ } else {
+ return namespace;
+ }
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
index b94711c..9bd7a12 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
@@ -18,11 +18,14 @@
package org.apache.hadoop.yarn.server.timeline;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -40,6 +43,8 @@
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespaces;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
@@ -59,6 +64,10 @@
new HashMap();
private Map entityInsertTimes =
new HashMap();
+ private Map namespacesById =
+ new HashMap();
+ private Map> namespacesByOwner =
+ new HashMap>();
public MemoryTimelineStore() {
super(MemoryTimelineStore.class.getName());
@@ -209,6 +218,58 @@ public TimelineEvents getEntityTimelines(String entityType,
}
@Override
+ public TimelineNamespace getNamespace(String namespaceId)
+ throws IOException {
+ TimelineNamespace namespace = namespacesById.get(namespaceId);
+ if (namespace == null) {
+ return null;
+ } else {
+ return createTimelineNamespace(
+ namespace.getId(),
+ namespace.getDescription(),
+ namespace.getOwner(),
+ namespace.getReaders(),
+ namespace.getWriters(),
+ namespace.getCreatedTime(),
+ namespace.getModifiedTime());
+ }
+ }
+
+ @Override
+ public TimelineNamespaces getNamespaces(String owner)
+ throws IOException {
+ List namespaces = new ArrayList();
+ for (TimelineNamespace namespace : namespacesByOwner.get(owner)) {
+ TimelineNamespace namespaceToReturn = createTimelineNamespace(
+ namespace.getId(),
+ namespace.getDescription(),
+ namespace.getOwner(),
+ namespace.getReaders(),
+ namespace.getWriters(),
+ namespace.getCreatedTime(),
+ namespace.getModifiedTime());
+ namespaces.add(namespaceToReturn);
+ }
+ Collections.sort(namespaces, new Comparator() {
+ @Override
+ public int compare(
+ TimelineNamespace namespace1, TimelineNamespace namespace2) {
+ int result = namespace2.getCreatedTime().compareTo(
+ namespace1.getCreatedTime());
+ if (result == 0) {
+ return namespace2.getModifiedTime().compareTo(
+ namespace1.getModifiedTime());
+ } else {
+ return result;
+ }
+ }
+ });
+ TimelineNamespaces namespacesToReturn = new TimelineNamespaces();
+ namespacesToReturn.addNamespaces(namespaces);
+ return namespacesToReturn;
+ }
+
+ @Override
public TimelinePutResponse put(TimelineEntities data) {
TimelinePutResponse response = new TimelinePutResponse();
for (TimelineEntity entity : data.getEntities()) {
@@ -306,6 +367,44 @@ public TimelinePutResponse put(TimelineEntities data) {
return response;
}
+ public void put(TimelineNamespace namespace) throws IOException {
+ TimelineNamespace namespaceToReplace =
+ namespacesById.get(namespace.getId());
+ long currentTimestamp = System.currentTimeMillis();
+ TimelineNamespace namespaceToStore = createTimelineNamespace(
+ namespace.getId(), namespace.getDescription(), namespace.getOwner(),
+ namespace.getReaders(), namespace.getWriters(),
+ (namespaceToReplace == null ?
+ currentTimestamp : namespaceToReplace.getCreatedTime()),
+ currentTimestamp);
+ namespacesById.put(namespaceToStore.getId(), namespaceToStore);
+ Set namespacesByOneOwner =
+ namespacesByOwner.get(namespaceToStore.getOwner());
+ if (namespacesByOneOwner == null) {
+ namespacesByOneOwner = new HashSet();
+ namespacesByOwner.put(namespaceToStore.getOwner(), namespacesByOneOwner);
+ }
+ if (namespaceToReplace != null) {
+ namespacesByOneOwner.remove(namespaceToReplace);
+ }
+ namespacesByOneOwner.add(namespaceToStore);
+ }
+
+ private static TimelineNamespace createTimelineNamespace(
+ String id, String description, String owner,
+ String readers, String writers,
+ Long createdTime, Long modifiedTime) {
+ TimelineNamespace namespaceToStore = new TimelineNamespace();
+ namespaceToStore.setId(id);
+ namespaceToStore.setDescription(description);
+ namespaceToStore.setOwner(owner);
+ namespaceToStore.setReaders(readers);
+ namespaceToStore.setWriters(writers);
+ namespaceToStore.setCreatedTime(createdTime);
+ namespaceToStore.setModifiedTime(modifiedTime);
+ return namespaceToStore;
+ }
+
private static TimelineEntity maskFields(
TimelineEntity entity, EnumSet fields) {
// Conceal the fields that are not going to be exposed
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java
index e68e860..217fbe4 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java
@@ -34,6 +34,8 @@
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespaces;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
@@ -286,6 +288,78 @@ public TimelinePutResponse postEntities(
return response;
}
+ /**
+ * Add or update an namespace. If the namespace already exists, only the owner
+ * and the admin can update it.
+ */
+ public void putNamespace(TimelineNamespace namespace,
+ UserGroupInformation callerUGI) throws YarnException, IOException {
+ TimelineNamespace existingNamespace =
+ store.getNamespace(namespace.getId());
+ if (existingNamespace != null) {
+ if (!timelineACLsManager.checkAccess(callerUGI, existingNamespace)) {
+ throw new YarnException(callerUGI.getShortUserName() +
+ " is not allowed to override an existing namespace " +
+ existingNamespace.getId());
+ }
+ // Set it again in case ACLs are not enabled: The namespace can be
+ // modified by every body, but the owner is not changed.
+ namespace.setOwner(existingNamespace.getOwner());
+ }
+ store.put(namespace);
+ }
+
+ /**
+ * Get a single namespace of the particular ID. If callerUGI is not the owner
+ * or the admin of the namespace, we need to hide the details from him, and
+ * only allow him to see the ID.
+ */
+ public TimelineNamespace getNamspace(String namespaceId,
+ UserGroupInformation callerUGI) throws YarnException, IOException {
+ TimelineNamespace namespace = store.getNamespace(namespaceId);
+ if (namespace != null) {
+ if (timelineACLsManager.checkAccess(callerUGI, namespace)) {
+ return namespace;
+ } else {
+ hideNamespaceDetails(namespace);
+ return namespace;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Get all the namespaces that belong to the given owner. If callerUGI is not
+ * the owner or the admin of the namespace, we need to hide the details from
+ * him, and only allow him to see the ID.
+ */
+ public TimelineNamespaces getNamespaces(String owner,
+ UserGroupInformation callerUGI) throws YarnException, IOException {
+ TimelineNamespaces namespaces = store.getNamespaces(owner);
+ boolean hasAccess = true;
+ boolean isChecked = false;
+ for (TimelineNamespace namespace : namespaces.getNamespaces()) {
+ // The owner for each namespace is the same, just need to check on
+ if (!isChecked) {
+ hasAccess = timelineACLsManager.checkAccess(callerUGI, namespace);
+ isChecked = true;
+ }
+ if (!hasAccess) {
+ hideNamespaceDetails(namespace);
+ }
+ }
+ return namespaces;
+ }
+
+ private static void hideNamespaceDetails(TimelineNamespace namespace) {
+ namespace.setDescription(null);
+ namespace.setOwner(null);
+ namespace.setReaders(null);
+ namespace.setWriters(null);
+ namespace.setCreatedTime(null);
+ namespace.setModifiedTime(null);
+ }
+
private static boolean extendFields(EnumSet fieldEnums) {
boolean modified = false;
if (fieldEnums != null && !fieldEnums.contains(Field.PRIMARY_FILTERS)) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java
index 23bca34..589255a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java
@@ -29,6 +29,8 @@
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespaces;
/**
* This interface is for retrieving timeline information.
@@ -152,4 +154,25 @@ TimelineEntity getEntity(String entityId, String entityType, EnumSet
TimelineEvents getEntityTimelines(String entityType,
SortedSet entityIds, Long limit, Long windowStart,
Long windowEnd, Set eventTypes) throws IOException;
+
+ /**
+ * This method retrieves the namespace information for a given ID.
+ *
+ * @return a {@link TimelineNamespace} object.
+ * @throws IOException
+ */
+ TimelineNamespace getNamespace(
+ String namespaceId) throws IOException;
+
+ /**
+ * This method retrieves all the namespaces that belong to a given owner.
+ * The namespaces are sorted according to the created time firstly and the
+ * modified time secondly in descending order.
+ *
+ * @param owner
+ * the namespace owner
+ * @return an {@link TimelineNamespaces} object.
+ * @throws IOException
+ */
+ TimelineNamespaces getNamespaces(String owner) throws IOException;
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java
index a3e5aeb..76486e1 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java
@@ -18,13 +18,14 @@
package org.apache.hadoop.yarn.server.timeline;
+import java.io.IOException;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
-import java.io.IOException;
-
/**
* This interface is for storing timeline information.
*/
@@ -37,10 +38,21 @@
* individual put request objects will be reported in the response.
*
* @param data
- * An {@link TimelineEntities} object.
- * @return An {@link TimelinePutResponse} object.
+ * a {@link TimelineEntities} object.
+ * @return a {@link TimelinePutResponse} object.
* @throws IOException
*/
TimelinePutResponse put(TimelineEntities data) throws IOException;
+ /**
+ * Store namespace information to the timeline store. If A namespace of the
+ * same ID already exists in the timeline store, it will be COMPLETELY updated
+ * with the given namespace.
+ *
+ * @param namespace
+ * a {@link TimelineNamespace} object
+ * @throws IOException
+ */
+ void put(TimelineNamespace namespace) throws IOException;
+
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java
index 10e62d2..4f6a99a 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java
@@ -27,6 +27,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AdminACLsManager;
import org.apache.hadoop.yarn.server.timeline.EntityIdentifier;
@@ -81,6 +82,31 @@ public boolean checkAccess(UserGroupInformation callerUGI,
return false;
}
+ public boolean checkAccess(UserGroupInformation callerUGI,
+ TimelineNamespace namespace) throws YarnException, IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Verifying the access of "
+ + (callerUGI == null ? null : callerUGI.getShortUserName())
+ + " on the timeline namespace " + namespace);
+ }
+
+ if (!adminAclsManager.areACLsEnabled()) {
+ return true;
+ }
+
+ String owner = namespace.getOwner();
+ if (owner == null || owner.length() == 0) {
+ throw new YarnException("Owner information of the timeline namespace "
+ + namespace.getId() + " is corrupted.");
+ }
+ if (callerUGI != null
+ && (adminAclsManager.isAdmin(callerUGI) ||
+ callerUGI.getShortUserName().equals(owner))) {
+ return true;
+ }
+ return false;
+ }
+
@Private
@VisibleForTesting
public AdminACLsManager
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
index c5e6d49..93f77ee 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.timeline.webapp;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
@@ -32,6 +33,7 @@
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
@@ -40,6 +42,7 @@
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
@@ -53,7 +56,10 @@
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespaces;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.timeline.EntityIdentifier;
import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.apache.hadoop.yarn.server.timeline.NameValuePair;
@@ -259,6 +265,100 @@ public TimelinePutResponse postEntities(
}
}
+ /**
+ * Store the given namespace into the timeline store, and return the errors
+ * that happen during storing.
+ */
+ @PUT
+ @Path("/namespace")
+ @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public Response putNamespace(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ TimelineNamespace namespace) {
+ init(res);
+ UserGroupInformation callerUGI = getUser(req);
+ if (callerUGI == null) {
+ String msg = "The owner of the posted timeline namespace is not set";
+ LOG.error(msg);
+ throw new ForbiddenException(msg);
+ }
+ namespace.setOwner(callerUGI.getShortUserName());
+ try {
+ timelineDataManager.putNamespace(namespace, callerUGI);
+ } catch (YarnException e) {
+ // The user doesn't have the access to override the existing namespace.
+ LOG.error(e.getMessage(), e);
+ throw new ForbiddenException(e);
+ } catch (IOException e) {
+ LOG.error("Error putting namespace", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ return Response.status(Status.OK).build();
+ }
+
+ /**
+ * Return a single namespace of the given namespace Id.
+ */
+ @GET
+ @Path("/namespace/{namespaceId}")
+ @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public TimelineNamespace getNamespace(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("namespaceId") String namespaceId) {
+ init(res);
+ namespaceId = parseStr(namespaceId);
+ if (namespaceId == null || namespaceId.length() == 0) {
+ throw new BadRequestException("Namespace ID is not specified.");
+ }
+ TimelineNamespace namespace = null;
+ try {
+ namespace = timelineDataManager.getNamspace(
+ parseStr(namespaceId), getUser(req));
+ } catch (Exception e) {
+ LOG.error("Error getting namespace", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ if (namespace == null) {
+ throw new NotFoundException("Timeline namespace ["
+ + namespaceId + "] is not found");
+ }
+ return namespace;
+ }
+
+ /**
+ * Return a list of namespaces of the given owner.
+ */
+ @GET
+ @Path("/namespace")
+ @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public TimelineNamespaces getNamespaces(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @QueryParam("owner") String owner) {
+ init(res);
+ owner = parseStr(owner);
+ UserGroupInformation callerUGI = getUser(req);
+ if (owner == null || owner.length() == 0) {
+ if (callerUGI == null) {
+ throw new BadRequestException("Namespace owner is not specified.");
+ } else {
+ // By default it's going to list the caller's namespaces
+ owner = callerUGI.getShortUserName();
+ }
+ }
+ try {
+ return timelineDataManager.getNamespaces(owner, callerUGI);
+ } catch (Exception e) {
+ LOG.error("Error getting namespaces", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
private void init(HttpServletResponse response) {
response.setContentType(null);
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
index 0c6e082..b752bcc 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
@@ -69,8 +69,9 @@ public void setup() throws Exception {
store = new LeveldbTimelineStore();
store.init(config);
store.start();
- loadTestData();
- loadVerificationData();
+ loadTestEntityData();
+ loadVerificationEntityData();
+ loadTestNamespaceData();
}
@After
@@ -93,7 +94,7 @@ public void testGetSingleEntity() throws IOException {
super.testGetSingleEntity();
((LeveldbTimelineStore)store).clearStartTimeCache();
super.testGetSingleEntity();
- loadTestData();
+ loadTestEntityData();
}
@Test
@@ -257,7 +258,7 @@ public void testFromTsWithDeletion()
assertEquals(0, getEntities("type_2").size());
assertEquals(0, getEntitiesFromTsWithPrimaryFilter("type_1", userFilter,
l).size());
- loadTestData();
+ loadTestEntityData();
assertEquals(0, getEntitiesFromTs("type_1", l).size());
assertEquals(0, getEntitiesFromTs("type_2", l).size());
assertEquals(0, getEntitiesFromTsWithPrimaryFilter("type_1", userFilter,
@@ -309,4 +310,14 @@ private void restartTimelineStore() throws IOException {
store.start();
}
+ @Test
+ public void testGetNamespace() throws IOException {
+ super.testGetNamespace();
+ }
+
+ @Test
+ public void testGetNamespaces() throws IOException {
+ super.testGetNamespaces();
+ }
+
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java
index 1953442..22921f2 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java
@@ -34,8 +34,9 @@ public void setup() throws Exception {
store = new MemoryTimelineStore();
store.init(new YarnConfiguration());
store.start();
- loadTestData();
- loadVerificationData();
+ loadTestEntityData();
+ loadVerificationEntityData();
+ loadTestNamespaceData();
}
@After
@@ -82,4 +83,14 @@ public void testGetEvents() throws IOException {
super.testGetEvents();
}
+ @Test
+ public void testGetNamespace() throws IOException {
+ super.testGetNamespace();
+ }
+
+ @Test
+ public void testGetNamespaces() throws IOException {
+ super.testGetNamespaces();
+ }
+
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java
index e8a6d83..da955f4 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java
@@ -38,11 +38,11 @@
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespaces;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
-import org.apache.hadoop.yarn.server.timeline.NameValuePair;
-import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
public class TimelineStoreTestUtils {
@@ -88,9 +88,9 @@
protected long beforeTs;
/**
- * Load test data into the given store
+ * Load test entity data into the given store
*/
- protected void loadTestData() throws IOException {
+ protected void loadTestEntityData() throws IOException {
beforeTs = System.currentTimeMillis()-1;
TimelineEntities entities = new TimelineEntities();
Map> primaryFilters =
@@ -184,9 +184,9 @@ protected void loadTestData() throws IOException {
}
/**
- * Load verification data
+ * Load verification entity data
*/
- protected void loadVerificationData() throws Exception {
+ protected void loadVerificationEntityData() throws Exception {
userFilter = new NameValuePair("user", "username");
numericFilter1 = new NameValuePair("appname", Integer.MAX_VALUE);
numericFilter2 = new NameValuePair("long", (long)Integer.MAX_VALUE + 1l);
@@ -263,6 +263,51 @@ protected void loadVerificationData() throws Exception {
events2.add(ev4);
}
+ private TimelineNamespace namespace1;
+ private TimelineNamespace namespace2;
+ private TimelineNamespace namespace3;
+ private long elapsedTime;
+
+ protected void loadTestNamespaceData() throws IOException {
+ namespace1 = new TimelineNamespace();
+ namespace1.setId("namespace_id_1");
+ namespace1.setDescription("description_1");
+ namespace1.setOwner("owner_1");
+ namespace1.setReaders("reader_user_1 reader_group_1");
+ namespace1.setWriters("writer_user_1 writer_group_1");
+ store.put(namespace1);
+
+ namespace2 = new TimelineNamespace();
+ namespace2.setId("namespace_id_2");
+ namespace2.setDescription("description_2");
+ namespace2.setOwner("owner_2");
+ namespace2.setReaders("reader_user_2 reader_group_2");
+ namespace2.setWriters("writer_user_2writer_group_2");
+ store.put(namespace2);
+
+ // Wait a second before updating the namespace information
+ elapsedTime = 1000;
+ try {
+ Thread.sleep(elapsedTime);
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+
+ namespace2.setDescription("description_3");
+ namespace2.setOwner("owner_3");
+ namespace2.setReaders("reader_user_3 reader_group_3");
+ namespace2.setWriters("writer_user_3 writer_group_3");
+ store.put(namespace2);
+
+ namespace3 = new TimelineNamespace();
+ namespace3.setId("namespace_id_4");
+ namespace3.setDescription("description_4");
+ namespace3.setOwner("owner_1");
+ namespace3.setReaders("reader_user_4 reader_group_4");
+ namespace3.setWriters("writer_user_4 writer_group_4");
+ store.put(namespace3);
+ }
+
public void testGetSingleEntity() throws IOException {
// test getting entity info
verifyEntityInfo(null, null, null, null, null, null,
@@ -519,7 +564,7 @@ public void testGetEntitiesWithFromTs() throws IOException {
assertEquals(2, getEntitiesWithPrimaryFilter("type_1", userFilter).size());
// check insert time is not overwritten
long beforeTs = this.beforeTs;
- loadTestData();
+ loadTestEntityData();
assertEquals(0, getEntitiesFromTs("type_1", beforeTs).size());
assertEquals(0, getEntitiesFromTs("type_2", beforeTs).size());
assertEquals(0, getEntitiesFromTsWithPrimaryFilter("type_1", userFilter,
@@ -788,4 +833,39 @@ private static TimelineEvent createEvent(long timestamp, String type, Map 0);
+ assertTrue(actualNamespace1.getModifiedTime() > 0);
+ assertEquals(
+ actualNamespace1.getCreatedTime(), actualNamespace1.getModifiedTime());
+
+ TimelineNamespace actualNamespace2 =
+ store.getNamespace(namespace2.getId());
+ verifyNamespaceInfo(namespace2, actualNamespace2);
+ assertEquals("namespace_id_2", actualNamespace2.getId());
+ assertTrue(actualNamespace2.getCreatedTime() > 0);
+ assertTrue(actualNamespace2.getModifiedTime() > 0);
+ assertTrue(
+ actualNamespace2.getCreatedTime() < actualNamespace2.getModifiedTime());
+ }
+
+ public void testGetNamespaces() throws IOException {
+ TimelineNamespaces actualNamespaces =
+ store.getNamespaces("owner_1");
+ assertEquals(2, actualNamespaces.getNamespaces().size());
+ verifyNamespaceInfo(namespace3, actualNamespaces.getNamespaces().get(0));
+ verifyNamespaceInfo(namespace1, actualNamespaces.getNamespaces().get(1));
+ }
+
+ private static void verifyNamespaceInfo(
+ TimelineNamespace expected, TimelineNamespace actual) {
+ assertEquals(expected.getId(), actual.getId());
+ assertEquals(expected.getDescription(), actual.getDescription());
+ assertEquals(expected.getOwner(), actual.getOwner());
+ assertEquals(expected.getReaders(), actual.getReaders());
+ assertEquals(expected.getWriters(), actual.getWriters());
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java
index 5825e7e..2e64db7 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java
@@ -21,17 +21,17 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
-import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
import org.junit.Assert;
import org.junit.Test;
public class TestTimelineACLsManager {
@Test
- public void testYarnACLsNotEnabled() throws Exception {
+ public void testYarnACLsNotEnabledForEntity() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
TimelineACLsManager timelineACLsManager =
@@ -47,7 +47,7 @@ public void testYarnACLsNotEnabled() throws Exception {
}
@Test
- public void testYarnACLsEnabled() throws Exception {
+ public void testYarnACLsEnabledForEntity() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
@@ -72,7 +72,7 @@ public void testYarnACLsEnabled() throws Exception {
}
@Test
- public void testCorruptedOwnerInfo() throws Exception {
+ public void testCorruptedOwnerInfoForEntity() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner");
@@ -89,4 +89,59 @@ public void testCorruptedOwnerInfo() throws Exception {
}
}
+ @Test
+ public void testYarnACLsNotEnabledForNamespace() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
+ TimelineACLsManager timelineACLsManager =
+ new TimelineACLsManager(conf);
+ TimelineNamespace namespace = new TimelineNamespace();
+ namespace.setOwner("owner");
+ Assert.assertTrue(
+ "Always true when ACLs are not enabled",
+ timelineACLsManager.checkAccess(
+ UserGroupInformation.createRemoteUser("user"), namespace));
+ }
+
+ @Test
+ public void testYarnACLsEnabledForNamespace() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
+ TimelineACLsManager timelineACLsManager =
+ new TimelineACLsManager(conf);
+ TimelineNamespace namespace = new TimelineNamespace();
+ namespace.setOwner("owner");
+ Assert.assertTrue(
+ "Owner should be allowed to access",
+ timelineACLsManager.checkAccess(
+ UserGroupInformation.createRemoteUser("owner"), namespace));
+ Assert.assertFalse(
+ "Other shouldn't be allowed to access",
+ timelineACLsManager.checkAccess(
+ UserGroupInformation.createRemoteUser("other"), namespace));
+ Assert.assertTrue(
+ "Admin should be allowed to access",
+ timelineACLsManager.checkAccess(
+ UserGroupInformation.createRemoteUser("admin"), namespace));
+ }
+
+ @Test
+ public void testCorruptedOwnerInfoForNamespace() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner");
+ TimelineACLsManager timelineACLsManager =
+ new TimelineACLsManager(conf);
+ TimelineNamespace namespace = new TimelineNamespace();
+ try {
+ timelineACLsManager.checkAccess(
+ UserGroupInformation.createRemoteUser("owner"), namespace);
+ Assert.fail("Exception is expected");
+ } catch (YarnException e) {
+ Assert.assertTrue("It's not the exact expected exception", e.getMessage()
+ .contains("is corrupted."));
+ }
+ }
+
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
index 549cfe1..fad78f7 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
@@ -36,6 +36,7 @@
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response.Status;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
@@ -44,6 +45,8 @@
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespace;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineNamespaces;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -69,7 +72,6 @@
import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
-
public class TestTimelineWebServices extends JerseyTest {
private static TimelineStore store;
@@ -85,7 +87,7 @@ protected void configureServlets() {
bind(YarnJacksonJaxbJsonProvider.class);
bind(TimelineWebServices.class);
bind(GenericExceptionHandler.class);
- try{
+ try {
store = mockTimelineStore();
} catch (Exception e) {
Assert.fail();
@@ -100,7 +102,8 @@ protected void configureServlets() {
new TimelineDataManager(store, timelineACLsManager);
bind(TimelineDataManager.class).toInstance(timelineDataManager);
serve("/*").with(GuiceContainer.class);
- TimelineAuthenticationFilter taFilter = new TimelineAuthenticationFilter();
+ TimelineAuthenticationFilter taFilter =
+ new TimelineAuthenticationFilter();
FilterConfig filterConfig = mock(FilterConfig.class);
when(filterConfig.getInitParameter(AuthenticationFilter.CONFIG_PREFIX))
.thenReturn(null);
@@ -159,7 +162,8 @@ public TestTimelineWebServices() {
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter")
.servletPath("/")
- .clientConfig(new DefaultClientConfig(YarnJacksonJaxbJsonProvider.class))
+ .clientConfig(
+ new DefaultClientConfig(YarnJacksonJaxbJsonProvider.class))
.build());
}
@@ -277,7 +281,7 @@ public void testPrimaryFilterLong() {
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline")
.path("type_1").queryParam("primaryFilter",
- "long:" + Long.toString((long)Integer.MAX_VALUE + 1l))
+ "long:" + Long.toString((long) Integer.MAX_VALUE + 1l))
.accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
@@ -406,7 +410,8 @@ public void testPostEntitiesWithPrimaryFilter() throws Exception {
TimelineEntities entities = new TimelineEntities();
TimelineEntity entity = new TimelineEntity();
Map> filters = new HashMap>();
- filters.put(TimelineStore.SystemFilter.ENTITY_OWNER.toString(), new HashSet());
+ filters.put(TimelineStore.SystemFilter.ENTITY_OWNER.toString(),
+ new HashSet());
entity.setPrimaryFilters(filters);
entity.setEntityId("test id 6");
entity.setEntityType("test type 6");
@@ -418,13 +423,15 @@ public void testPostEntitiesWithPrimaryFilter() throws Exception {
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.post(ClientResponse.class, entities);
- TimelinePutResponse putResposne = response.getEntity(TimelinePutResponse.class);
+ TimelinePutResponse putResposne =
+ response.getEntity(TimelinePutResponse.class);
Assert.assertEquals(1, putResposne.getErrors().size());
List errors = putResposne.getErrors();
- Assert.assertEquals(TimelinePutResponse.TimelinePutError.SYSTEM_FILTER_CONFLICT,
- errors.get(0).getErrorCode());
+ Assert.assertEquals(
+ TimelinePutResponse.TimelinePutError.SYSTEM_FILTER_CONFLICT,
+ errors.get(0).getErrorCode());
}
-
+
@Test
public void testPostEntities() throws Exception {
TimelineEntities entities = new TimelineEntities();
@@ -449,7 +456,8 @@ public void testPostEntities() throws Exception {
.type(MediaType.APPLICATION_JSON)
.post(ClientResponse.class, entities);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
- TimelinePutResponse putResposne = response.getEntity(TimelinePutResponse.class);
+ TimelinePutResponse putResposne =
+ response.getEntity(TimelinePutResponse.class);
Assert.assertNotNull(putResposne);
Assert.assertEquals(0, putResposne.getErrors().size());
// verify the entity exists in the store
@@ -482,7 +490,8 @@ public void testPostEntitiesWithYarnACLsEnabled() throws Exception {
.type(MediaType.APPLICATION_JSON)
.post(ClientResponse.class, entities);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
- TimelinePutResponse putResponse = response.getEntity(TimelinePutResponse.class);
+ TimelinePutResponse putResponse =
+ response.getEntity(TimelinePutResponse.class);
Assert.assertNotNull(putResponse);
Assert.assertEquals(0, putResponse.getErrors().size());
@@ -668,4 +677,202 @@ public void testGetEventsWithYarnACLsEnabled() {
}
}
+ @Test
+ public void testGetNamespace() throws Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("timeline")
+ .path("namespace").path("namespace_id_1")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ TimelineNamespace namespace = response.getEntity(TimelineNamespace.class);
+ verifyNamespace(namespace, "namespace_id_1", true);
+ }
+
+ @Test
+ public void testGetNamespaceYarnACLsEnabled() {
+ AdminACLsManager oldAdminACLsManager =
+ timelineACLsManager.setAdminACLsManager(adminACLsManager);
+ try {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("timeline")
+ .path("namespace").path("namespace_id_1")
+ .queryParam("user.name", "owner_1")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ TimelineNamespace namespace = response.getEntity(TimelineNamespace.class);
+ verifyNamespace(namespace, "namespace_id_1", true);
+
+ response = r.path("ws").path("v1").path("timeline")
+ .path("namespace").path("namespace_id_1")
+ .queryParam("user.name", "tester")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ namespace = response.getEntity(TimelineNamespace.class);
+ verifyNamespace(namespace, "namespace_id_1", false);
+ } finally {
+ timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
+ }
+ }
+
+ @Test
+ public void testGetNamespaces() throws Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("timeline")
+ .path("namespace")
+ .queryParam("owner", "owner_1")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ TimelineNamespaces namespaces = response.getEntity(TimelineNamespaces.class);
+ Assert.assertEquals(2, namespaces.getNamespaces().size());
+ for (int i = 0; i < namespaces.getNamespaces().size(); ++i) {
+ verifyNamespace(namespaces.getNamespaces().get(i),
+ i == 0 ? "namespace_id_4" : "namespace_id_1", true);
+ }
+ }
+
+ @Test
+ public void testGetNamespacesYarnACLsEnabled() throws Exception {
+ AdminACLsManager oldAdminACLsManager =
+ timelineACLsManager.setAdminACLsManager(adminACLsManager);
+ try {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("timeline")
+ .path("namespace")
+ .queryParam("user.name", "owner_1")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ TimelineNamespaces namespaces = response.getEntity(TimelineNamespaces.class);
+ Assert.assertEquals(2, namespaces.getNamespaces().size());
+ for (int i = 0; i < namespaces.getNamespaces().size(); ++i) {
+ verifyNamespace(namespaces.getNamespaces().get(i),
+ i == 0 ? "namespace_id_4" : "namespace_id_1", true);
+ }
+
+ response = r.path("ws").path("v1").path("timeline")
+ .path("namespace")
+ .queryParam("owner", "owner_1")
+ .queryParam("user.name", "tester")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ namespaces = response.getEntity(TimelineNamespaces.class);
+ Assert.assertEquals(2, namespaces.getNamespaces().size());
+ for (int i = 0; i < namespaces.getNamespaces().size(); ++i) {
+ verifyNamespace(namespaces.getNamespaces().get(i),
+ i == 0 ? "namespace_id_4" : "namespace_id_1", false);
+ }
+ } finally {
+ timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
+ }
+ }
+
+ @Test
+ public void testPutNamespace() throws Exception {
+ TimelineNamespace namespace = new TimelineNamespace();
+ namespace.setId("test_namespace_id");
+ WebResource r = resource();
+ // No owner, will be rejected
+ ClientResponse response = r.path("ws").path("v1")
+ .path("timeline").path("namespace")
+ .accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class, namespace);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ assertEquals(ClientResponse.Status.FORBIDDEN,
+ response.getClientResponseStatus());
+
+ response = r.path("ws").path("v1")
+ .path("timeline").path("namespace")
+ .queryParam("user.name", "tester")
+ .accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class, namespace);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+
+ // Verify the namespace exists
+ response = r.path("ws").path("v1").path("timeline")
+ .path("namespace").path("test_namespace_id")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ namespace = response.getEntity(TimelineNamespace.class);
+ Assert.assertNotNull(namespace);
+ Assert.assertEquals("test_namespace_id", namespace.getId());
+ Assert.assertEquals("tester", namespace.getOwner());
+ Assert.assertEquals(null, namespace.getDescription());
+
+ // Update the namespace
+ namespace.setDescription("test_description");
+ response = r.path("ws").path("v1")
+ .path("timeline").path("namespace")
+ .queryParam("user.name", "tester")
+ .accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class, namespace);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+
+ // Verify the namespace is updated
+ response = r.path("ws").path("v1").path("timeline")
+ .path("namespace").path("test_namespace_id")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ namespace = response.getEntity(TimelineNamespace.class);
+ Assert.assertNotNull(namespace);
+ Assert.assertEquals("test_namespace_id", namespace.getId());
+ Assert.assertEquals("test_description", namespace.getDescription());
+ }
+
+ @Test
+ public void testPutNamespaceYarnACLsEnabled() throws Exception {
+ AdminACLsManager oldAdminACLsManager =
+ timelineACLsManager.setAdminACLsManager(adminACLsManager);
+ try {
+ TimelineNamespace namespace = new TimelineNamespace();
+ namespace.setId("test_namespace_id_acl");
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1")
+ .path("timeline").path("namespace")
+ .queryParam("user.name", "tester")
+ .accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class, namespace);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+
+ // Update the namespace by another user
+ response = r.path("ws").path("v1")
+ .path("timeline").path("namespace")
+ .queryParam("user.name", "other")
+ .accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class, namespace);
+ assertEquals(Status.FORBIDDEN.getStatusCode(), response.getStatus());
+ } finally {
+ timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
+ }
+ }
+
+ private static void verifyNamespace(TimelineNamespace namespace,
+ String namespaceId, boolean hasAccess) {
+ Assert.assertNotNull(namespace);
+ Assert.assertEquals(namespaceId, namespace.getId());
+ // The specific values have been verified in TestMemoryTimelineStore
+ Assert.assertTrue(hasAccess && namespace.getDescription() != null ||
+ !hasAccess && namespace.getDescription() == null);
+ Assert.assertTrue(hasAccess && namespace.getOwner() != null ||
+ !hasAccess && namespace.getOwner() == null);
+ Assert.assertTrue(hasAccess && namespace.getReaders() != null ||
+ !hasAccess && namespace.getReaders() == null);
+ Assert.assertTrue(hasAccess && namespace.getWriters() != null ||
+ !hasAccess && namespace.getWriters() == null);
+ Assert.assertTrue(hasAccess && namespace.getCreatedTime() != null ||
+ !hasAccess && namespace.getCreatedTime() == null);
+ Assert.assertTrue(hasAccess && namespace.getModifiedTime() != null ||
+ !hasAccess && namespace.getModifiedTime() == null);
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java
index 81f87fb..7c1fe16 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java
@@ -24,7 +24,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
@@ -124,8 +123,8 @@ public void testPutEntities() throws Exception {
private ClientResponse resp;
@Override
- public ClientResponse doPostingEntities(TimelineEntities entities) {
- resp = super.doPostingEntities(entities);
+ public ClientResponse doPostingObject(Object obj, String path) {
+ resp = super.doPostingObject(obj, path);
return resp;
}