commit 79158648b8c53ff71686528e9bf2d9218d7d3dc9 Author: eyang Date: Fri Oct 6 16:26:49 2017 -0400 YARN-7215. Added REST API to list deployed services by user. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml index 153d198..8fe0ad9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml @@ -26,6 +26,10 @@ jar Hadoop YARN REST APIs for services + + 6.0.0 + + @@ -65,6 +69,15 @@ + + org.apache.rat + apache-rat-plugin + + + **/*.json + + + @@ -151,5 +164,46 @@ test + + org.apache.solr + solr-solrj + ${solr.version} + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + slf4j-api + + + + + + org.apache.solr + solr-core + ${solr.version} + + + org.eclipse.jetty + jetty-xml + + + test + + + + org.apache.solr + solr-test-framework + ${solr.version} + + + org.eclipse.jetty + jetty-xml + + + test + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index 2c903cc..059c3b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -34,6 +34,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.naming.ServiceUnavailableException; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; @@ -47,6 +48,7 @@ import javax.ws.rs.core.Response.Status; import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Map; import static org.apache.hadoop.yarn.service.api.records.ServiceState.ACCEPTED; @@ -59,29 +61,34 @@ @Path(CONTEXT_ROOT) public class ApiServer { + private static final Logger LOG = + LoggerFactory.getLogger(ApiServer.class); + protected Configuration YARN_CONFIG; + protected ServiceClient SERVICE_CLIENT; + public ApiServer() { super(); + YARN_CONFIG = new YarnConfiguration(); + try { + SERVICE_CLIENT = new ServiceClient(); + SERVICE_CLIENT.init(YARN_CONFIG); + SERVICE_CLIENT.start(); + } catch (Exception e) { + LOG.error("Fail to initialize ServiceClient, ApiServer is unavailable."); + } } @Inject public ApiServer(Configuration conf) { super(); - } - - private static final Logger LOG = - LoggerFactory.getLogger(ApiServer.class); - private static Configuration YARN_CONFIG = new YarnConfiguration(); - private static ServiceClient SERVICE_CLIENT; - - static { - init(); - } - - // initialize all the common resources - order is important - private static void init() { - SERVICE_CLIENT = new ServiceClient(); - SERVICE_CLIENT.init(YARN_CONFIG); - SERVICE_CLIENT.start(); + YARN_CONFIG = conf; + try { + SERVICE_CLIENT = new ServiceClient(); + SERVICE_CLIENT.init(YARN_CONFIG); + SERVICE_CLIENT.start(); + } catch (Exception e) { + LOG.error("Fail to initialize ServiceClient, ApiServer is unavailable."); + } } @GET @@ -124,12 +131,31 @@ public Response createService(Service service) { } @GET + @Path(SERVICE_ROOT_PATH) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response getServicesList() { + LOG.info("GET: getServicesList"); + ServiceStatus serviceStatus = new ServiceStatus(); + try { + List list = SERVICE_CLIENT.getServicesList(); + return Response.ok().entity(list).build(); + } catch (IOException | ServiceUnavailableException e) { + LOG.error("Get services list failed", e); + serviceStatus + .setDiagnostics("Failed to retrieve service: " + e.getMessage()); + return Response.status(Status.SERVICE_UNAVAILABLE) + .entity(serviceStatus).build(); + } + } + + @GET @Path(SERVICE_SPEC_PATH) @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response getServiceSpec(@PathParam(SERVICE_NAME) String appName) { - LOG.info("GET: getServiceConfig for appName = {}", appName); + LOG.info("GET: getServiceSpec for appName = {}", appName); ServiceStatus serviceStatus = new ServiceStatus(); try { Service app = SERVICE_CLIENT.getSpec(appName); @@ -374,9 +400,10 @@ private Response startService(String appName) { * * @param mockServerClient - A mocked version of ServiceClient */ - public static void setApiServer(ServiceClient mockServerClient) { + public void setServiceClient(ServiceClient mockServerClient, + Configuration conf) { SERVICE_CLIENT = mockServerClient; - SERVICE_CLIENT.init(YARN_CONFIG); + YARN_CONFIG = conf; SERVICE_CLIENT.start(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java index f74bcd0..61d2d47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java @@ -21,10 +21,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.client.ServiceClient; +import org.apache.hadoop.yarn.service.client.YarnSolrClientTest; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; /** @@ -34,13 +36,15 @@ */ public class ServiceClientTest extends ServiceClient { - private Configuration conf = new Configuration(); + private Configuration conf = new YarnConfiguration(); protected static void init() { } public ServiceClientTest() { super(); + this.init(conf); + createYarnSolrClient(); } @Override @@ -49,6 +53,11 @@ public Configuration getConfig() { } @Override + protected void createYarnSolrClient() { + ysc = new YarnSolrClientTest(); + } + + @Override public ApplicationId actionCreate(Service service) { String serviceName = service.getName(); ServiceApiUtil.validateNameFormat(serviceName, getConfig()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java index 34af930..734a3ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.service; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.service.api.records.Artifact; import org.apache.hadoop.yarn.service.api.records.Artifact.TypeEnum; import org.apache.hadoop.yarn.service.api.records.Component; @@ -25,12 +26,15 @@ import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ServiceState; import org.apache.hadoop.yarn.service.client.ServiceClient; +import org.apache.hadoop.yarn.service.client.YarnSolrClientTest; import org.apache.hadoop.yarn.service.conf.RestApiConstants; import org.apache.hadoop.yarn.service.webapp.ApiServer; + import javax.ws.rs.Path; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -50,13 +54,20 @@ @Before public void setup() throws Exception { - ServiceClient mockServerClient = new ServiceClientTest(); - Configuration conf = new Configuration(); + Configuration conf = new YarnConfiguration(); conf.set("yarn.api-service.service.client.class", ServiceClientTest.class.getName()); conf.setBoolean(RestApiConstants.YARN_SOLR_STORAGE_ENABLED, true); - ApiServer.setApiServer(mockServerClient); - this.apiServer = new ApiServer(conf); + ServiceClient mockServerClient = new ServiceClientTest(); + apiServer = new ApiServer(conf); + apiServer.setServiceClient(mockServerClient, conf); + } + + @After + public void teardown() { + Configuration conf = new Configuration(); + YarnSolrClientTest ysct = new YarnSolrClientTest(); + ysct.cleanup(); } @Test @@ -382,4 +393,16 @@ public void testUpdateService() { is(Response.status(Status.INTERNAL_SERVER_ERROR) .build().getStatus())); } + + /** + * Default configuration disables SOLR storage, list services + * REST API should return service unavailable message. + */ + @Test + public void testGetServicesList() { + final Response actual = apiServer.getServicesList(); + assertThat("List services is disabled.", actual.getStatus(), + is(Response.status(Status.SERVICE_UNAVAILABLE) + .build().getStatus())); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiService.java index f4d96d5..803392b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiService.java @@ -16,7 +16,6 @@ */ package org.apache.hadoop.yarn.service; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.service.api.records.Artifact; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Resource; @@ -24,6 +23,7 @@ import org.apache.hadoop.yarn.service.api.records.Artifact.TypeEnum; import org.apache.hadoop.yarn.service.client.ServiceClient; import org.apache.hadoop.yarn.service.webapp.ApiServer; +import org.apache.hadoop.yarn.service.client.ServiceTestUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -35,11 +35,9 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.Is.is; import static org.junit.Assert.*; -import static org.apache.hadoop.yarn.service.ServiceTestUtils.NUM_NMS; +import static org.apache.hadoop.yarn.service.client.ServiceTestUtils.NUM_NMS; /** * End to end api service test. @@ -50,11 +48,8 @@ @Before public void setup() throws Exception { - ServiceClient mockServerClient = new ServiceClientTest(); - Configuration conf = new Configuration(); - ApiServer.setApiServer(mockServerClient); - this.apiServer = new ApiServer(conf); setupInternal(NUM_NMS); + apiServer = createApiServer(); } @After @@ -85,11 +80,11 @@ public void testCreateServiceAndCheckSpec() { components.add(c); service.setComponents(components); final Response actual = apiServer.createService(service); - assertThat("Create service is ", actual.getStatus(), - is(Response.status(Status.ACCEPTED).build().getStatus())); + assertEquals("Create service is ", actual.getStatus(), + Response.status(Status.ACCEPTED).build().getStatus()); final Response actual2 = apiServer.getServiceSpec("jenkins"); - assertThat("Get service spec is ", actual2.getStatus(), - is(Response.status(Status.OK).build().getStatus())); + assertEquals("Get service spec is ", actual2.getStatus(), + Response.status(Status.OK).build().getStatus()); } /** @@ -116,13 +111,22 @@ public void testFlexService() { components.add(c); service.setComponents(components); final Response actual = apiServer.createService(service); - assertThat("Create service is ", actual.getStatus(), - is(Response.status(Status.ACCEPTED).build().getStatus())); + assertEquals("Create service is ", actual.getStatus(), + Response.status(Status.ACCEPTED).build().getStatus()); for(Component c2 : service.getComponents()) { c2.setNumberOfContainers(2L); } final Response actual2 = apiServer.updateService("jenkins", service); - assertThat("Flex service is ", actual.getStatus(), - is(Response.status(Status.ACCEPTED).build().getStatus())); + assertEquals("Flex service is ", actual2.getStatus(), + Response.status(Status.NO_CONTENT).build().getStatus()); } + + private ApiServer createApiServer() throws Exception { + ServiceClient sc = createClient(); + ApiServer api = new ApiServer() { + protected ServiceClient SERVICE_CLIENT = sc; + }; + return api; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt new file mode 100644 index 0000000..2c164c0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +# Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +such +that +the +their +then +there +these +they +this +to +was +will +with diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/params.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/params.json new file mode 100644 index 0000000..06114ef --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/params.json @@ -0,0 +1,20 @@ +{"params":{ + "query":{ + "defType":"edismax", + "q.alt":"*:*", + "rows":"10", + "fl":"*,score", + "":{"v":0} + }, + "facets":{ + "facet":"on", + "facet.mincount": "1", + "":{"v":0} + }, + "velocity":{ + "wt": "velocity", + "v.template":"browse", + "v.layout": "layout", + "":{"v":0} + } +}} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/protwords.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/protwords.txt new file mode 100644 index 0000000..1dfc0ab --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/schema.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/schema.xml new file mode 100644 index 0000000..90dc4b8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/schema.xml @@ -0,0 +1,116 @@ + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml new file mode 100644 index 0000000..392feec --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml @@ -0,0 +1,36 @@ + + + + 6.2.1 + + ${solr.data.dir:} + + + + + single + + + + + + + + + + \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/stopwords.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/stopwords.txt new file mode 100644 index 0000000..ae1e83e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/stopwords.txt @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/synonyms.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/synonyms.txt new file mode 100644 index 0000000..7f72128 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/synonyms.txt @@ -0,0 +1,29 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaafoo => aaabar +bbbfoo => bbbfoo bbbbar +cccfoo => cccbar cccbaz +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index f2bccb3..59a6ff7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -28,7 +28,7 @@ ${project.parent.basedir} - 6.3.0 + 6.0.0 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 9edb1e6..06b0ed1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -107,6 +107,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import javax.naming.ServiceUnavailableException; + import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*; import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_CREATE; import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_FLEX; @@ -854,6 +856,19 @@ public ServiceState convertState(FinalApplicationStatus status) { } /** + * Fetch list of deployed services from SOLR. + * + * @return list of services + * @throws ServiceUnavailableException + * @throws IOException + */ + public List getServicesList() + throws ServiceUnavailableException, IOException { + List list = ysc.listAppEntry(); + return list; + } + + /** * Fetch service configuration from SOLR. * * @param serviceName - Service name diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/YarnSolrClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/YarnSolrClient.java index 77c72b2..e884c90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/YarnSolrClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/YarnSolrClient.java @@ -19,9 +19,13 @@ package org.apache.hadoop.yarn.service.client; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; +import java.util.List; + +import javax.naming.ServiceUnavailableException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -37,17 +41,22 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.conf.RestApiConstants; public class YarnSolrClient { + /** + * Storage driver for persist Yarnfile on SOLR. + */ private static final Log LOG = LogFactory.getLog(YarnSolrClient.class); private static String SOLR_URL; private static final String DEFAULT_SOLR_URL = "http://localhost:8983/solr/yarn"; private static boolean ENABLED; + private static int MAXFILTERLIMIT = 40; public YarnSolrClient() { super(); @@ -66,7 +75,7 @@ private static void init() { } public SolrClient getSolrClient() { - return new HttpSolrClient.Builder(SOLR_URL).build(); + return new HttpSolrClient(SOLR_URL); } public Service findAppEntry(String appName, String username) { @@ -149,4 +158,45 @@ public void deleteApp(String id, String username) { } } + public List listAppEntry() + throws ServiceUnavailableException, IOException { + String userName = ""; + List list = new ArrayList(); + if (!ENABLED) { + throw new ServiceUnavailableException("YARN Solr " + + "support is unavailable."); + } + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, + false); + + SolrClient solr = getSolrClient(); + SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + query.setFilterQueries("type_s:AppEntry"); + if (UserGroupInformation.isSecurityEnabled()) { + userName = UserGroupInformation.getCurrentUser().getUserName(); + if (!userName.isEmpty()) { + query.addFilterQuery("user_s:" + userName); + } + } + query.setRows(MAXFILTERLIMIT); + + QueryResponse response; + try { + response = solr.query(query); + Iterator appList = response.getResults() + .listIterator(); + while (appList.hasNext()) { + SolrDocument d = appList.next(); + Service entry = mapper.readValue(d.get("yarnfile_s").toString(), + Service.class); + list.add(entry); + } + } catch (SolrServerException | IOException e) { + LOG.error("Error in finding service for user: " + userName, e); + } + return list; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java deleted file mode 100644 index 33a9146..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service; - -import org.apache.commons.io.FileUtils; -import org.apache.curator.test.TestingCluster; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.yarn.service.api.records.Service; -import org.apache.hadoop.yarn.service.conf.YarnServiceConf; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.MiniYARNCluster; -import org.apache.hadoop.yarn.service.api.records.Component; -import org.apache.hadoop.yarn.service.api.records.Resource; -import org.apache.hadoop.yarn.service.utils.JsonSerDeser; -import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; -import org.apache.hadoop.yarn.service.utils.SliderFileSystem; -import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin; -import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree; -import org.codehaus.jackson.map.PropertyNamingStrategy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.net.URL; - -import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_PMEM_CHECK_ENABLED; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_VMEM_CHECK_ENABLED; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_SERVICE_ENABLED; -import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.AM_RESOURCE_MEM; -import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; -import static org.mockito.Matchers.anyObject; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Base utility to test ServiceClient class. - */ -public class ServiceTestUtils { - - private static final Logger LOG = - LoggerFactory.getLogger(ServiceTestUtils.class); - - private MiniYARNCluster yarnCluster = null; - private MiniDFSCluster hdfsCluster = null; - private FileSystem fs = null; - private Configuration conf = null; - public static final int NUM_NMS = 1; - private File basedir; - - public static final JsonSerDeser JSON_SER_DESER = - new JsonSerDeser<>(Service.class, - PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); - - // Example service definition - // 2 components, each of which has 2 containers. - protected Service createExampleApplication() { - Service exampleApp = new Service(); - exampleApp.setName("example-app"); - exampleApp.addComponent(createComponent("compa")); - exampleApp.addComponent(createComponent("compb")); - return exampleApp; - } - - protected Component createComponent(String name) { - return createComponent(name, 2L, "sleep 1000"); - } - - protected Component createComponent(String name, long numContainers, - String command) { - Component comp1 = new Component(); - comp1.setNumberOfContainers(numContainers); - comp1.setLaunchCommand(command); - comp1.setName(name); - Resource resource = new Resource(); - comp1.setResource(resource); - resource.setMemory("128"); - resource.setCpus(1); - return comp1; - } - - public static SliderFileSystem initMockFs() throws IOException { - return initMockFs(null); - } - - public static SliderFileSystem initMockFs(Service ext) throws IOException { - SliderFileSystem sfs = mock(SliderFileSystem.class); - FileSystem mockFs = mock(FileSystem.class); - JsonSerDeser jsonSerDeser = mock(JsonSerDeser.class); - when(sfs.getFileSystem()).thenReturn(mockFs); - when(sfs.buildClusterDirPath(anyObject())).thenReturn( - new Path("cluster_dir_path")); - if (ext != null) { - when(jsonSerDeser.load(anyObject(), anyObject())).thenReturn(ext); - } - ServiceApiUtil.setJsonSerDeser(jsonSerDeser); - return sfs; - } - - protected void setConf(YarnConfiguration conf) { - this.conf = conf; - } - - protected Configuration getConf() { - return conf; - } - - protected FileSystem getFS() { - return fs; - } - - protected void setupInternal(int numNodeManager) - throws Exception { - LOG.info("Starting up YARN cluster"); -// Logger rootLogger = LogManager.getRootLogger(); -// rootLogger.setLevel(Level.DEBUG); - setConf(new YarnConfiguration()); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); - // reduce the teardown waiting time - conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000); - conf.set("yarn.log.dir", "target"); - // mark if we need to launch the v1 timeline server - // disable aux-service based timeline aggregators - conf.set(YarnConfiguration.NM_AUX_SERVICES, ""); - conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8"); - // Enable ContainersMonitorImpl - conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR, - LinuxResourceCalculatorPlugin.class.getName()); - conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, - ProcfsBasedProcessTree.class.getName()); - conf.setBoolean( - YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, true); - conf.setBoolean(TIMELINE_SERVICE_ENABLED, false); - conf.setInt(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100); - conf.setLong(DEBUG_NM_DELETE_DELAY_SEC, 60000); - conf.setLong(AM_RESOURCE_MEM, 526); - conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 5); - // Disable vmem check to disallow NM killing the container - conf.setBoolean(NM_VMEM_CHECK_ENABLED, false); - conf.setBoolean(NM_PMEM_CHECK_ENABLED, false); - // setup zk cluster - TestingCluster zkCluster; - zkCluster = new TestingCluster(1); - zkCluster.start(); - conf.set(YarnConfiguration.RM_ZK_ADDRESS, zkCluster.getConnectString()); - conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString()); - LOG.info("ZK cluster: " + zkCluster.getConnectString()); - - fs = FileSystem.get(conf); - basedir = new File("target", "apps"); - if (basedir.exists()) { - FileUtils.deleteDirectory(basedir); - } else { - basedir.mkdirs(); - } - - conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath()); - - if (yarnCluster == null) { - yarnCluster = - new MiniYARNCluster(TestYarnNativeServices.class.getSimpleName(), 1, - numNodeManager, 1, 1); - yarnCluster.init(conf); - yarnCluster.start(); - - waitForNMsToRegister(); - - URL url = Thread.currentThread().getContextClassLoader() - .getResource("yarn-site.xml"); - if (url == null) { - throw new RuntimeException( - "Could not find 'yarn-site.xml' dummy file in classpath"); - } - Configuration yarnClusterConfig = yarnCluster.getConfig(); - yarnClusterConfig.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, - new File(url.getPath()).getParent()); - //write the document to a buffer (not directly to the file, as that - //can cause the file being written to get read -which will then fail. - ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); - yarnClusterConfig.writeXml(bytesOut); - bytesOut.close(); - //write the bytes to the file in the classpath - OutputStream os = new FileOutputStream(new File(url.getPath())); - os.write(bytesOut.toByteArray()); - os.close(); - LOG.info("Write yarn-site.xml configs to: " + url); - } - if (hdfsCluster == null) { - HdfsConfiguration hdfsConfig = new HdfsConfiguration(); - hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig) - .numDataNodes(1).build(); - } - - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - LOG.info("setup thread sleep interrupted. message=" + e.getMessage()); - } - } - - public void shutdown() throws IOException { - if (yarnCluster != null) { - try { - yarnCluster.stop(); - } finally { - yarnCluster = null; - } - } - if (hdfsCluster != null) { - try { - hdfsCluster.shutdown(); - } finally { - hdfsCluster = null; - } - } - if (basedir != null) { - FileUtils.deleteDirectory(basedir); - } - SliderFileSystem sfs = new SliderFileSystem(conf); - Path appDir = sfs.getBaseApplicationPath(); - sfs.getFileSystem().delete(appDir, true); - } - - private void waitForNMsToRegister() throws Exception { - int sec = 60; - while (sec >= 0) { - if (yarnCluster.getResourceManager().getRMContext().getRMNodes().size() - >= NUM_NMS) { - break; - } - Thread.sleep(1000); - sec--; - } - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java index 959e4d6..7f10a25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java @@ -21,6 +21,7 @@ import org.apache.hadoop.registry.client.api.RegistryConstants; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.client.ServiceTestUtils; import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages; import org.apache.hadoop.yarn.service.api.records.Artifact; import org.apache.hadoop.yarn.service.api.records.Component; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java index 4898dea..8aa0800 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java @@ -23,15 +23,13 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Container; import org.apache.hadoop.yarn.service.api.records.ContainerState; import org.apache.hadoop.yarn.service.client.ServiceClient; -import org.apache.hadoop.yarn.service.client.YarnSolrClientTest; -import org.apache.hadoop.yarn.service.exceptions.SliderException; +import org.apache.hadoop.yarn.service.client.ServiceTestUtils; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; import org.junit.After; import org.junit.Assert; @@ -304,24 +302,7 @@ private void waitForAllCompToBeReady(ServiceClient client, }, 2000, 200000); } - private ServiceClient createClient() throws Exception { - ServiceClient client = new ServiceClient() { - @Override - protected Path addJarResource(String appName, - Map localResources) - throws IOException, SliderException { - // do nothing, the Unit test will use local jars - return null; - } - @Override - protected void createYarnSolrClient() { - ysc = new YarnSolrClientTest(); - } - }; - client.init(getConf()); - client.start(); - return client; - } + private int countTotalContainers(Service service) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/EmbeddedSolrServerFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/EmbeddedSolrServerFactory.java index 59f0cf4..63ef492 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/EmbeddedSolrServerFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/EmbeddedSolrServerFactory.java @@ -29,7 +29,13 @@ import java.nio.file.Path; import java.nio.file.Paths; -public class EmbeddedSolrServerFactory { +/** + * Unit test utility class for testing SOLR. + */ +public final class EmbeddedSolrServerFactory { + + private EmbeddedSolrServerFactory() { + } /** * Cleans the given solrHome directory and creates a new diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/ServiceTestUtils.java new file mode 100644 index 0000000..894daf7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/ServiceTestUtils.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.client; + +import org.apache.commons.io.FileUtils; +import org.apache.curator.test.TestingCluster; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.client.ServiceClient; +import org.apache.hadoop.yarn.service.conf.YarnServiceConf; +import org.apache.hadoop.yarn.service.exceptions.SliderException; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.MiniYARNCluster; +import org.apache.hadoop.yarn.service.TestYarnNativeServices; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.Resource; +import org.apache.hadoop.yarn.service.utils.JsonSerDeser; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; +import org.apache.hadoop.yarn.service.utils.SliderFileSystem; +import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin; +import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree; +import org.codehaus.jackson.map.PropertyNamingStrategy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URL; +import java.util.Map; + +import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_PMEM_CHECK_ENABLED; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_VMEM_CHECK_ENABLED; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_SERVICE_ENABLED; +import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.AM_RESOURCE_MEM; +import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; +import static org.apache.hadoop.yarn.service.conf.RestApiConstants.YARN_SOLR_STORAGE_ENABLED; +import static org.mockito.Matchers.anyObject; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Base utility to test ServiceClient class. + */ +public class ServiceTestUtils { + + private static final Logger LOG = + LoggerFactory.getLogger(ServiceTestUtils.class); + + private MiniYARNCluster yarnCluster = null; + private MiniDFSCluster hdfsCluster = null; + private FileSystem fs = null; + public Configuration conf = null; + public static final int NUM_NMS = 1; + private File basedir; + + public static final JsonSerDeser JSON_SER_DESER = + new JsonSerDeser<>(Service.class, + PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); + + // Example service definition + // 2 components, each of which has 2 containers. + protected Service createExampleApplication() { + Service exampleApp = new Service(); + exampleApp.setName("example-app"); + exampleApp.addComponent(createComponent("compa")); + exampleApp.addComponent(createComponent("compb")); + return exampleApp; + } + + protected Component createComponent(String name) { + return createComponent(name, 2L, "sleep 1000"); + } + + protected Component createComponent(String name, long numContainers, + String command) { + Component comp1 = new Component(); + comp1.setNumberOfContainers(numContainers); + comp1.setLaunchCommand(command); + comp1.setName(name); + Resource resource = new Resource(); + comp1.setResource(resource); + resource.setMemory("128"); + resource.setCpus(1); + return comp1; + } + + public static SliderFileSystem initMockFs() throws IOException { + return initMockFs(null); + } + + public static SliderFileSystem initMockFs(Service ext) throws IOException { + SliderFileSystem sfs = mock(SliderFileSystem.class); + FileSystem mockFs = mock(FileSystem.class); + JsonSerDeser jsonSerDeser = mock(JsonSerDeser.class); + when(sfs.getFileSystem()).thenReturn(mockFs); + when(sfs.buildClusterDirPath(anyObject())).thenReturn( + new Path("cluster_dir_path")); + if (ext != null) { + when(jsonSerDeser.load(anyObject(), anyObject())).thenReturn(ext); + } + ServiceApiUtil.setJsonSerDeser(jsonSerDeser); + return sfs; + } + + protected void setConf(YarnConfiguration conf) { + this.conf = conf; + } + + protected Configuration getConf() { + return conf; + } + + protected FileSystem getFS() { + return fs; + } + + protected void setupInternal(int numNodeManager) + throws Exception { + LOG.info("Starting up YARN cluster"); +// Logger rootLogger = LogManager.getRootLogger(); +// rootLogger.setLevel(Level.DEBUG); + setConf(new YarnConfiguration()); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); + // reduce the teardown waiting time + conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000); + conf.set("yarn.log.dir", "target"); + // mark if we need to launch the v1 timeline server + // disable aux-service based timeline aggregators + conf.set(YarnConfiguration.NM_AUX_SERVICES, ""); + conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8"); + // Enable ContainersMonitorImpl + conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR, + LinuxResourceCalculatorPlugin.class.getName()); + conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, + ProcfsBasedProcessTree.class.getName()); + conf.setBoolean( + YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, true); + conf.setBoolean(TIMELINE_SERVICE_ENABLED, false); + conf.setInt(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100); + conf.setLong(DEBUG_NM_DELETE_DELAY_SEC, 60000); + conf.setLong(AM_RESOURCE_MEM, 526); + conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 5); + // Disable vmem check to disallow NM killing the container + conf.setBoolean(NM_VMEM_CHECK_ENABLED, false); + conf.setBoolean(NM_PMEM_CHECK_ENABLED, false); + conf.setBoolean(YARN_SOLR_STORAGE_ENABLED, true); + // setup zk cluster + TestingCluster zkCluster; + zkCluster = new TestingCluster(1); + zkCluster.start(); + conf.set(YarnConfiguration.RM_ZK_ADDRESS, zkCluster.getConnectString()); + conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString()); + LOG.info("ZK cluster: " + zkCluster.getConnectString()); + + fs = FileSystem.get(conf); + basedir = new File("target", "apps"); + if (basedir.exists()) { + FileUtils.deleteDirectory(basedir); + } else { + basedir.mkdirs(); + } + + conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath()); + + if (yarnCluster == null) { + yarnCluster = + new MiniYARNCluster(TestYarnNativeServices.class.getSimpleName(), 1, + numNodeManager, 1, 1); + yarnCluster.init(conf); + yarnCluster.start(); + + waitForNMsToRegister(); + + URL url = Thread.currentThread().getContextClassLoader() + .getResource("yarn-site.xml"); + if (url == null) { + throw new RuntimeException( + "Could not find 'yarn-site.xml' dummy file in classpath"); + } + Configuration yarnClusterConfig = yarnCluster.getConfig(); + yarnClusterConfig.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, + new File(url.getPath()).getParent()); + //write the document to a buffer (not directly to the file, as that + //can cause the file being written to get read -which will then fail. + ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); + yarnClusterConfig.writeXml(bytesOut); + bytesOut.close(); + //write the bytes to the file in the classpath + OutputStream os = new FileOutputStream(new File(url.getPath())); + os.write(bytesOut.toByteArray()); + os.close(); + LOG.info("Write yarn-site.xml configs to: " + url); + } + if (hdfsCluster == null) { + HdfsConfiguration hdfsConfig = new HdfsConfiguration(); + hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig) + .numDataNodes(1).build(); + } + + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + LOG.info("setup thread sleep interrupted. message=" + e.getMessage()); + } + } + + public void shutdown() throws IOException { + if (yarnCluster != null) { + try { + yarnCluster.stop(); + } finally { + yarnCluster = null; + } + } + if (hdfsCluster != null) { + try { + hdfsCluster.shutdown(); + } finally { + hdfsCluster = null; + } + } + if (basedir != null) { + FileUtils.deleteDirectory(basedir); + } + SliderFileSystem sfs = new SliderFileSystem(conf); + Path appDir = sfs.getBaseApplicationPath(); + sfs.getFileSystem().delete(appDir, true); + } + + private void waitForNMsToRegister() throws Exception { + int sec = 60; + while (sec >= 0) { + if (yarnCluster.getResourceManager().getRMContext().getRMNodes().size() + >= NUM_NMS) { + break; + } + Thread.sleep(1000); + sec--; + } + } + + public ServiceClient createClient() throws Exception { + ServiceClient client = new ServiceClient() { + @Override + protected Path addJarResource(String appName, + Map localResources) + throws IOException, SliderException { + // do nothing, the Unit test will use local jars + return null; + } + @Override + protected void createYarnSolrClient() { + ysc = new YarnSolrClientTest(); + } + }; + client.init(getConf()); + client.serviceInit(getConf()); + client.start(); + return client; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/YarnSolrClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/YarnSolrClientTest.java index 7c2d406..9ce5b96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/YarnSolrClientTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/YarnSolrClientTest.java @@ -18,18 +18,25 @@ package org.apache.hadoop.yarn.service.client; +import java.io.File; + import org.apache.solr.client.solrj.SolrClient; +/** + * Mocked version of Yarn Solr Client for testing with + * embedded SOLR server. + * + */ public class YarnSolrClientTest extends YarnSolrClient { static final String CONFIGSET_DIR = "src/test/resources/configsets"; - static SolrClient solrClient; + private static SolrClient solrClient; - static String targetLocation = EmbeddedSolrServerFactory.class + private static String targetLocation = EmbeddedSolrServerFactory.class .getProtectionDomain().getCodeSource().getLocation().getFile() + "/.."; - static String solrHome = targetLocation + "/solr"; + private static String solrHome = targetLocation + "/solr"; public YarnSolrClientTest() { super(); @@ -45,4 +52,11 @@ public SolrClient getSolrClient() { return solrClient; } + public void cleanup() { + File f = new File(solrHome); + if (f.exists()) { + f.delete(); + } + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java index 5fdd2ab..a4df02c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java @@ -21,12 +21,12 @@ import org.apache.hadoop.yarn.service.api.records.Service; +import static org.apache.hadoop.yarn.service.client.ServiceTestUtils.JSON_SER_DESER; + import java.io.IOException; import java.util.ArrayList; import java.util.List; -import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER; - /** * Names of the example configs. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java index 8739382..98cebba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java @@ -19,8 +19,8 @@ package org.apache.hadoop.yarn.service.conf; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.service.ServiceTestUtils; import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.client.ServiceTestUtils; import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.hadoop.yarn.service.api.records.Configuration; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java index a813da3..2085a2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java @@ -19,8 +19,8 @@ package org.apache.hadoop.yarn.service.conf; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.service.ServiceTestUtils; import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.client.ServiceTestUtils; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; import org.junit.Assert; @@ -28,11 +28,11 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import static org.apache.hadoop.yarn.service.client.ServiceTestUtils.JSON_SER_DESER; + import java.util.Arrays; import java.util.Collection; -import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER; - /** * Test loading example resources. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java index 0e03a2c..44f43b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java @@ -23,9 +23,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.service.MockServiceAM; -import org.apache.hadoop.yarn.service.ServiceTestUtils; - import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.client.ServiceTestUtils; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.conf.YarnServiceConf; import org.junit.After;