diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index df6c7a4..50968fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -35,13 +35,12 @@ import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLSocketFactory; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -67,6 +66,8 @@ import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; @@ -81,12 +82,14 @@ import com.sun.jersey.api.client.filter.ClientFilter; import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Private @Unstable public class TimelineClientImpl extends TimelineClient { - private static final Log LOG = LogFactory.getLog(TimelineClientImpl.class); + private static final Logger LOG = LoggerFactory.getLogger(TimelineClientImpl.class); private static final String RESOURCE_URI_STR = "/ws/v1/timeline/"; private static final Joiner JOINER = Joiner.on(""); public final static int DEFAULT_SOCKET_TIMEOUT = 1 * 60 * 1000; // 1 minute @@ -471,6 +474,54 @@ public ClientResponse doPostingObject(Object object, String path) { } } + @Private + @VisibleForTesting + public ClientResponse doGettingJson(String path, + MultivaluedMap queryParams) throws YarnException { + WebResource webResource = client.resource(resURI); + WebResource target = webResource.path(path).queryParams(queryParams); + String targetText = target.toString(); + if (path != null) { + ClientResponse response = null; + try { + response = target + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + } catch (RuntimeException re) { + // runtime exception is expected if the client cannot connect the server + String msg = + "Failed to get the response from the timeline server at " + targetText; + LOG.error(msg, re); + throw re; + } + if (response == null || + response.getClientResponseStatus() != ClientResponse.Status.OK) { + if (response != null) { + try { + JSONObject json = new JSONObject(response.getEntity(String.class)); + if (LOG.isDebugEnabled()) { + LOG.debug("GET {} => HTTP error code: {}; Server response : \n {}", + targetText, response.getStatus(), json.toString()); + } + throw new YarnException( + String.format("GET %s => HTTP error code %d - %s", + targetText, + response.getStatus(), + json.getString("message"))); + } catch (JSONException je) { + // couldn't grab the actual exception, so fall back to a generic + // -sounding one + } + } + String msg = "Failed to get the response from the timeline server at " + + targetText; + LOG.error(msg); + throw new YarnException(msg); + } + return response; + } + throw new YarnRuntimeException("Unknown resource type from " + targetText); + } + private class TimelineURLConnectionFactory implements HttpURLConnectionFactory { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestTimelineClientPut.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestTimelineClientPut.java new file mode 100644 index 0000000..c598647 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestTimelineClientPut.java @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timeline; + +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.core.util.MultivaluedMapImpl; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.service.ServiceOperations; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; +import org.apache.hadoop.yarn.client.api.TimelineClient; +import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.rules.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.core.MultivaluedMap; +import java.io.File; +import java.io.IOException; +import java.util.UUID; + +public class TestTimelineClientPut { + private static final Logger LOG = LoggerFactory.getLogger(TestTimelineClientPut.class); + + @Rule + public Timeout testTimeout = new Timeout(30000); + + @Rule + public TestName methodName = new TestName(); + + private TimelineClientImpl client; + private ApplicationHistoryServer ahs; + // better if this were scanned + private static int port = 10210; + + @BeforeClass + public static void nameThread() { + Thread.currentThread().setName("JUnit"); + } + + @After + public void tearDown() { + ServiceOperations.stopQuietly(client); + ServiceOperations.stopQuietly(ahs); + } + + @Test + public void testPutEntitiesInMemory() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, + MemoryTimelineStore.class, TimelineStore.class); + createClientAndAHS(conf); + + putEntities(); + getEntities(); + } + + @Test + public void testPutEntitiesInLDB() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, + LeveldbTimelineStore.class, TimelineStore.class); + FileContext fsContext = FileContext.getLocalFSFileContext(); + File fsPath = new File("target", + String.format("%s-%s-db", + this.getClass().getSimpleName(), methodName.getMethodName()) + ).getAbsoluteFile(); + fsContext.delete(new Path(fsPath.getAbsolutePath()), true); + conf.set(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH, + fsPath.getAbsolutePath()); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_TTL_ENABLE, false); + + conf.set(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH, + "target/"); + createClientAndAHS(conf); + + putEntities(); + getEntities(); + } + + + private void putEntities() throws IOException, YarnException { + TimelineEntity entity = new TimelineEntity(); + entity.setEntityType("test"); + entity.setEntityId("testid-" + UUID.randomUUID()); + entity.setStartTime(System.currentTimeMillis()); + entity.addPrimaryFilter("prop1", "val1"); + entity.addOtherInfo("info1", "val1"); + + + client.putEntities(entity); + + entity.addOtherInfo("info2", "val2"); + entity.addPrimaryFilter("prop2", "val2"); + client.putEntities(entity); + } + + /* + From YARN-2477 + The following search causes the problem described above: + /ws/v1/timeline/test?primaryFilter=prop2:val2 + The following one works as expected: + /ws/v1/timeline/test?primaryFilter=prop1:val1 + + */ + private void getEntities() throws YarnException { + // to work /ws/v1/timeline/test?primaryFilter=prop1:val1 + MultivaluedMap p1 = new MultivaluedMapImpl(); + p1.putSingle("primaryFilter", "prop1:val1"); + get(p1); + + // to fail /ws/v1/timeline/test?primaryFilter=prop2:val2 + MultivaluedMap p2 = new MultivaluedMapImpl(); + p1.putSingle("primaryFilter", "prop1:val2"); + get(p2); + } + + private ClientResponse get(MultivaluedMap queryParams) throws YarnException { + ClientResponse response = client.doGettingJson("/test", queryParams); + LOG.info(response.getEntity(String.class)); + return response; + } + + + private void createClientAndAHS(Configuration conf) { + conf.set(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, "127.0.0.1:" + port++); + conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "127.0.0.1:" + port++); + + + client = (TimelineClientImpl) TimelineClient.createTimelineClient(); + client.init(conf); + client.start(); + + ahs = new ApplicationHistoryServer(); + ahs.init(client.getConfig()); + ahs.start(); + } + +}