diff --git hadoop-project/pom.xml hadoop-project/pom.xml index 3a6519c..6a8fdfa 100644 --- hadoop-project/pom.xml +++ hadoop-project/pom.xml @@ -425,6 +425,11 @@ com.sun.jersey + jersey-client + ${jersey.version} + + + com.sun.jersey jersey-server ${jersey.version} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ea9b93a..cecc1ce 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1000,6 +1000,38 @@ AHS_PREFIX + "webapp.spnego-keytab-file"; //////////////////////////////// + // ATS Configs + //////////////////////////////// + + public static final String ATS_PREFIX = YARN_PREFIX + "ats."; + + // Redirect ATS web configurations to AHS as ATS REST APIs are hosted by + // AHS web server + /** The address of the ATS web application.*/ + public static final String ATS_WEBAPP_ADDRESS = AHS_WEBAPP_ADDRESS; + + public static final int DEFAULT_ATS_WEBAPP_PORT = DEFAULT_AHS_WEBAPP_PORT; + public static final String DEFAULT_ATS_WEBAPP_ADDRESS = + DEFAULT_AHS_WEBAPP_ADDRESS; + + /** The https address of the ATS web application.*/ + public static final String ATS_WEBAPP_HTTPS_ADDRESS = + AHS_WEBAPP_HTTPS_ADDRESS; + + public static final int DEFAULT_ATS_WEBAPP_HTTPS_PORT = + DEFAULT_AHS_WEBAPP_HTTPS_PORT; + public static final String DEFAULT_ATS_WEBAPP_HTTPS_ADDRESS = + DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS; + + /**The kerberos principal to be used for spnego filter for ATS.*/ + public static final String ATS_WEBAPP_SPNEGO_USER_NAME_KEY = + AHS_WEBAPP_SPNEGO_USER_NAME_KEY; + + /**The kerberos keytab to be used for spnego filter for ATS.*/ + public static final String ATS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = + AHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY; + + //////////////////////////////// // Other Configs //////////////////////////////// diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml index 54da659..6091686 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml @@ -79,6 +79,10 @@ org.mortbay.jetty jetty-util + + com.sun.jersey + jersey-client + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ATSClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ATSClient.java new file mode 100644 index 0000000..159f0bb --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ATSClient.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.client.api.impl.ATSClientImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; + +/** + * The client library that can be used to post the information of a number of + * conceptual entities of an application. + */ +@Public +@Unstable +public abstract class ATSClient extends AbstractService { + + @Public + public static ATSClient createAMATSClient() { + ATSClient client = new ATSClientImpl(); + return client; + } + + @Private + protected ATSClient(String name) { + super(name); + } + + /** + *

+ * Post the information of a number of conceptual entities of an application + * to the application timeline server via REST API. It is a blocking API. The + * method will not return until it gets the response from the application + * timeline server. + *

+ * + * @param entities + * the collection of {@link ATSEntity} + * @throws IOException + * @throws YarnException + */ + @Public + public abstract void postEntities( + ATSEntity... entities) throws IOException, YarnException; + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ATSClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ATSClientImpl.java new file mode 100644 index 0000000..52bb23c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ATSClientImpl.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api.impl; + +import java.io.IOException; +import java.net.URI; +import java.util.Arrays; + +import javax.ws.rs.core.MediaType; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.client.api.ATSClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.ClientConfig; +import com.sun.jersey.api.client.config.DefaultClientConfig; + +@Private +@Unstable +public class ATSClientImpl extends ATSClient { + + private static final Log LOG = LogFactory.getLog(ATSClientImpl.class); + private static final String RESOURCE_URI_STR = "/ws/v1/apptimeline/"; + private static final Joiner JOINER = Joiner.on(""); + + private Client client; + private URI resURI; + + public ATSClientImpl() { + super(ATSClientImpl.class.getName()); + ClientConfig cc = new DefaultClientConfig(); + cc.getClasses().add(YarnJacksonJaxbJsonProvider.class); + client = Client.create(cc); + } + + protected void serviceInit(Configuration conf) throws Exception { + resURI = new URI(JOINER.join(HttpConfig.getSchemePrefix(), + HttpConfig.isSecure() ? conf.get( + YarnConfiguration.ATS_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_ATS_WEBAPP_HTTPS_ADDRESS) : conf.get( + YarnConfiguration.ATS_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_ATS_WEBAPP_ADDRESS), RESOURCE_URI_STR)); + super.serviceInit(conf); + } + + @Override + public void postEntities( + ATSEntity... entities) throws IOException, YarnException { + ATSEntities atsEntities = new ATSEntities(); + atsEntities.addEntities(Arrays.asList(entities)); + ClientResponse resp = doPostingEntities(atsEntities); + if (resp.getStatus() != ClientResponse.Status.OK.getStatusCode()) { + String msg = "Failed to host application timeline entities. " + + "HTTP error code: " + resp.getStatus(); + LOG.error(msg); + if (LOG.isDebugEnabled()) { + String output = resp.getEntity(String.class); + LOG.debug("Server response : \n" + output); + } + throw new YarnException(msg); + } + } + + @Private + @VisibleForTesting + public ClientResponse doPostingEntities(ATSEntities atsEntities) { + WebResource webResource = client.resource(resURI); + return webResource.accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON) + .post(ClientResponse.class, atsEntities); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestATSClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestATSClient.java new file mode 100644 index 0000000..fb75bf6 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestATSClient.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api.impl; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import java.util.Arrays; + +import junit.framework.Assert; + +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity; +import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvent; +import org.apache.hadoop.yarn.client.api.ATSClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.sun.jersey.api.client.ClientResponse; + +public class TestATSClient { + + private ATSClientImpl client; + + @Before + public void setup() { + client = spy((ATSClientImpl) ATSClient.createAMATSClient()); + client.init(new YarnConfiguration()); + client.start(); + } + + @After + public void tearDown() { + client.stop(); + } + + @Test + public void testPostEntities() throws Exception { + mockClientResponse(ClientResponse.Status.OK.getStatusCode()); + try { + client.postEntities(generateATSEntity()); + } catch (YarnException e) { + Assert.fail("Exception is not expected"); + } + } + + @Test + public void testPostEntitiesError() throws Exception { + mockClientResponse( + ClientResponse.Status.INTERNAL_SERVER_ERROR.getStatusCode()); + try { + client.postEntities(generateATSEntity()); + Assert.fail("Exception is expected"); + } catch (YarnException e) { + Assert.assertTrue(e.getMessage().contains("HTTP error code: " + + ClientResponse.Status.INTERNAL_SERVER_ERROR.getStatusCode())); + } + } + + private ClientResponse mockClientResponse(int statusCode) { + ClientResponse response = mock(ClientResponse.class); + doReturn(response).when(client) + .doPostingEntities(any(ATSEntities.class)); + when(response.getStatus()).thenReturn(statusCode); + return response; + } + + private static ATSEntity generateATSEntity() { + ATSEntity entity = new ATSEntity(); + entity.setEntityId("entity id"); + entity.setEntityType("entity type"); + entity.setKeyTs(System.currentTimeMillis()); + for (int i = 0; i < 2; ++i) { + ATSEvent event = new ATSEvent(); + event.setTimestamp(System.currentTimeMillis()); + event.setEventType("test event type " + i); + event.addEventInfo("key1", "val1"); + event.addEventInfo("key2", "val2"); + entity.addEvent(event); + } + entity.addRelatedEntity( + "test ref type 1", Arrays.asList((Object) "test ref id 1")); + entity.addRelatedEntity( + "test ref type 2", Arrays.asList((Object) "test ref id 2")); + entity.addPrimaryFilter("pkey1", "pval1"); + entity.addPrimaryFilter("pkey2", "pval2"); + entity.addOtherInfo("okey1", "oval1"); + entity.addOtherInfo("okey2", "oval2"); + return entity; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java new file mode 100644 index 0000000..87ee7ec --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.webapp; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.ext.Provider; + +import org.codehaus.jackson.jaxrs.JacksonJaxbJsonProvider; +import org.codehaus.jackson.map.AnnotationIntrospector; +import org.codehaus.jackson.map.DeserializationConfig; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; +import org.codehaus.jackson.xc.JaxbAnnotationIntrospector; + +import com.google.inject.Singleton; + +@Singleton +@Provider +public class YarnJacksonJaxbJsonProvider extends JacksonJaxbJsonProvider { + + public YarnJacksonJaxbJsonProvider() { + super(); + } + + @SuppressWarnings("deprecation") + @Override + public ObjectMapper locateMapper(Class type, MediaType mediaType) { + ObjectMapper mapper = super.locateMapper(type, mediaType); + mapper.configure(DeserializationConfig.Feature.WRAP_ROOT_VALUE, true); + AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(); + //TODO: change to use non-deprecated methods + mapper.getSerializationConfig().setAnnotationIntrospector(introspector); + mapper.getDeserializationConfig().setAnnotationIntrospector(introspector); + mapper.getSerializationConfig() + .setSerializationInclusion(Inclusion.NON_NULL); + return mapper; + } +}