commit e21d87dc6cdc0bb57fe6213961c8fae861e50ada Author: eyang Date: Fri Oct 6 19:16:23 2017 -0400 YARN-7217, include YARN-7216, YARN-7215. Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/yarn-site.xml YARN-7215 Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiService.java YARN-7217. Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/yarn-site.xml Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 7e45e1a..c0e534d 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -434,6 +434,13 @@ org.apache.hadoop + hadoop-yarn-services-core + ${project.version} + test-jar + + + + org.apache.hadoop hadoop-mapreduce-client-jobclient ${project.version} test-jar diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 01c56b0..1a4669d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3333,6 +3333,12 @@ public static boolean areNodeLabelsEnabled( public static final String TIMELINE_XFS_OPTIONS = TIMELINE_XFS_PREFIX + "xframe-options"; + // YARN SOLR props + public static final String YARN_API_SERVICE_SOLR_STORAGE_ENABLED = + "yarn.api-service.solr.storage.enabled"; + public static final String YARN_API_SERVICE_SOLR_URL = + "yarn.api-service.solr.url"; + public YarnConfiguration() { super(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml index ddea2a1..5c1aab3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml @@ -26,6 +26,10 @@ jar Hadoop YARN REST APIs for services + + 6.0.0 + + @@ -65,6 +69,15 @@ + + org.apache.rat + apache-rat-plugin + + + **/*.json + + + @@ -126,5 +139,72 @@ test + + org.apache.hadoop + hadoop-hdfs + test-jar + + + org.apache.hadoop + hadoop-yarn-services-core + test-jar + + + org.apache.hadoop + hadoop-yarn-server-tests + test-jar + + + org.apache.hadoop + hadoop-minicluster + test + + + org.apache.curator + curator-test + test + + + + org.apache.solr + solr-solrj + ${solr.version} + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + slf4j-api + + + + + + org.apache.solr + solr-core + ${solr.version} + + + org.eclipse.jetty + jetty-xml + + + test + + + + org.apache.solr + solr-test-framework + ${solr.version} + + + org.eclipse.jetty + jetty-xml + + + test + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index 5773069..56d6a21 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -31,9 +31,11 @@ import org.apache.hadoop.yarn.service.api.records.ServiceStatus; import org.apache.hadoop.yarn.service.client.ServiceClient; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; +import org.apache.solr.client.solrj.SolrServerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.naming.ServiceUnavailableException; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; @@ -47,6 +49,7 @@ import javax.ws.rs.core.Response.Status; import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Map; import static org.apache.hadoop.yarn.service.api.records.ServiceState.ACCEPTED; @@ -59,29 +62,60 @@ @Path(CONTEXT_ROOT) public class ApiServer { + private static final Logger LOG = + LoggerFactory.getLogger(ApiServer.class); + private Configuration YARN_CONFIG; + private ServiceClient SERVICE_CLIENT; + + protected Configuration getConf() { + return YARN_CONFIG; + } + + protected void setConf(Configuration conf) { + YARN_CONFIG = conf; + } + /** + * Constructor used by standalone API Server. + */ public ApiServer() { - super(); + YARN_CONFIG = new YarnConfiguration(); + try { + if (SERVICE_CLIENT==null) { + SERVICE_CLIENT = new ServiceClient(); + SERVICE_CLIENT.init(YARN_CONFIG); + SERVICE_CLIENT.start(); + } + } catch (Exception e) { + LOG.error("Fail to initialize ServiceClient, ApiServer is unavailable."); + } } + /** + * Constructor used by ResourceManager. + * + * @param conf + */ @Inject public ApiServer(Configuration conf) { - super(); - } - - private static final Logger LOG = - LoggerFactory.getLogger(ApiServer.class); - private static Configuration YARN_CONFIG = new YarnConfiguration(); - private static ServiceClient SERVICE_CLIENT; - - static { - init(); + YARN_CONFIG = conf; + try { + if (SERVICE_CLIENT==null) { + SERVICE_CLIENT = new ServiceClient(); + SERVICE_CLIENT.init(YARN_CONFIG); + SERVICE_CLIENT.start(); + } + } catch (Exception e) { + LOG.error("Fail to initialize ServiceClient, ApiServer is unavailable."); + } } - // initialize all the common resources - order is important - private static void init() { - SERVICE_CLIENT = new ServiceClient(); - SERVICE_CLIENT.init(YARN_CONFIG); - SERVICE_CLIENT.start(); + /** + * Constructor used unit test. + * + * @param client + */ + public ApiServer(ServiceClient client) { + SERVICE_CLIENT = client; } @GET @@ -124,11 +158,57 @@ public Response createService(Service service) { } @GET - @Path(SERVICE_PATH) + @Path(SERVICE_ROOT_PATH) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response getServicesList() { + LOG.info("GET: getServicesList"); + ServiceStatus serviceStatus = new ServiceStatus(); + try { + List list = SERVICE_CLIENT.getServicesList(); + return Response.ok().entity(list).build(); + } catch (IOException | ServiceUnavailableException | + SolrServerException e) { + LOG.error("Get services list failed", e); + serviceStatus + .setDiagnostics("Failed to retrieve service: " + e.getMessage()); + return Response.status(Status.SERVICE_UNAVAILABLE) + .entity(serviceStatus).build(); + } + } + + @GET + @Path(SERVICE_SPEC_PATH) @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) - public Response getService(@PathParam(SERVICE_NAME) String appName) { - LOG.info("GET: getService for appName = {}", appName); + public Response getServiceSpec(@PathParam(SERVICE_NAME) + String appName) { + LOG.info("GET: getServiceSpec for appName = {}", appName); + ServiceStatus serviceStatus = new ServiceStatus(); + try { + Service app = SERVICE_CLIENT.getSpec(appName); + return Response.ok(app).build(); + } catch (IllegalArgumentException e) { + serviceStatus.setDiagnostics(e.getMessage()); + serviceStatus.setCode(ERROR_CODE_APP_NAME_INVALID); + return Response.status(Status.NOT_FOUND).entity(serviceStatus) + .build(); + } catch (Exception e) { + LOG.error("Get service failed", e); + serviceStatus + .setDiagnostics("Failed to retrieve service: " + e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(serviceStatus).build(); + } + } + + @GET + @Path(SERVICE_STATUS_PATH) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response getServiceStatus(@PathParam(SERVICE_NAME) + String appName) { + LOG.info("GET: getServiceStatus for appName = {}", appName); ServiceStatus serviceStatus = new ServiceStatus(); try { Service app = SERVICE_CLIENT.getStatus(appName); @@ -210,10 +290,10 @@ public Response updateComponent(@PathParam(SERVICE_NAME) String appName, } @PUT - @Path(SERVICE_PATH) + @Path(SERVICE_STATUS_PATH) @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) - public Response updateService(@PathParam(SERVICE_NAME) String appName, + public Response updateServiceState(@PathParam(SERVICE_NAME) String appName, Service updateServiceData) { LOG.info("PUT: updateService for app = {} with data = {}", appName, updateServiceData); @@ -294,14 +374,40 @@ private Response startService(String appName) { } /** - * Used by negative test case. + * Update service specification. * - * @param mockServerClient - A mocked version of ServiceClient + * @param appName + * @param updateServiceData + * @return */ - public static void setServiceClient(ServiceClient mockServerClient) { - SERVICE_CLIENT = mockServerClient; - SERVICE_CLIENT.init(YARN_CONFIG); - SERVICE_CLIENT.start(); + @PUT + @Path(SERVICE_SPEC_PATH) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response updateServiceSpec(@PathParam(SERVICE_NAME) + String serviceName, Service updateServiceData) { + ServiceStatus serviceStatus = new ServiceStatus(); + try { + Service service = SERVICE_CLIENT.updateSpec(serviceName, + updateServiceData); + return Response.ok(service).build(); + } catch (IllegalArgumentException e) { + serviceStatus.setDiagnostics(e.getMessage()); + serviceStatus.setCode(ERROR_CODE_APP_NAME_INVALID); + return Response.status(Status.NOT_FOUND).entity(serviceStatus) + .build(); + } catch (IOException e) { + serviceStatus + .setDiagnostics("Failed to update service:" + e.getMessage()); + return Response.status(Status.FORBIDDEN) + .entity(serviceStatus).build(); + } catch (Exception e) { + LOG.error("Update service failed", e); + serviceStatus + .setDiagnostics("Failed to update service: " + e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(serviceStatus).build(); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java deleted file mode 100644 index 3e08c3a..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.service.api.records.Service; -import org.apache.hadoop.yarn.service.client.ServiceClient; -import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; - -/** - * A mock version of ServiceClient - This class is design - * to simulate various error conditions that will happen - * when a consumer class calls ServiceClient. - */ -public class ServiceClientTest extends ServiceClient { - - private Configuration conf = new Configuration(); - - protected static void init() { - } - - public ServiceClientTest() { - super(); - } - - @Override - public Configuration getConfig() { - return conf; - } - - @Override - public ApplicationId actionCreate(Service service) { - String serviceName = service.getName(); - ServiceApiUtil.validateNameFormat(serviceName, getConfig()); - return ApplicationId.newInstance(System.currentTimeMillis(), 1); - } - - @Override - public Service getStatus(String appName) { - if (appName == null) { - throw new NullPointerException(); - } - if (appName.equals("jenkins")) { - return new Service(); - } else { - throw new IllegalArgumentException(); - } - } - - @Override - public int actionStart(String serviceName) - throws YarnException, IOException { - if (serviceName == null) { - throw new NullPointerException(); - } - if (serviceName.equals("jenkins")) { - return EXIT_SUCCESS; - } else { - throw new ApplicationNotFoundException(""); - } - } - - @Override - public int actionStop(String serviceName, boolean waitForAppStopped) - throws YarnException, IOException { - if (serviceName == null) { - throw new NullPointerException(); - } - if (serviceName.equals("jenkins")) { - return EXIT_SUCCESS; - } else { - throw new ApplicationNotFoundException(""); - } - } - - @Override - public int actionDestroy(String serviceName) { - if (serviceName == null) { - throw new NullPointerException(); - } - if (serviceName.equals("jenkins")) { - return EXIT_SUCCESS; - } else { - throw new IllegalArgumentException(); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java index 2b22474..798519c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.service; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.service.api.records.Artifact; import org.apache.hadoop.yarn.service.api.records.Artifact.TypeEnum; import org.apache.hadoop.yarn.service.api.records.Component; @@ -25,63 +26,107 @@ import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ServiceState; import org.apache.hadoop.yarn.service.client.ServiceClient; +import org.apache.hadoop.yarn.service.client.ServiceClientBadTest; +import org.apache.hadoop.yarn.service.client.YarnSolrClientTest; import org.apache.hadoop.yarn.service.webapp.ApiServer; + import javax.ws.rs.Path; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; -import org.junit.Before; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStream; +import java.net.URL; import java.util.ArrayList; import java.util.List; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.*; import static org.junit.Assert.*; /** - * Test case for ApiServer REST API. + * Test cases for ApiServer REST API. * */ public class TestApiServer { - private ApiServer apiServer; - - @Before - public void setup() throws Exception { - ServiceClient mockServerClient = new ServiceClientTest(); - Configuration conf = new Configuration(); - conf.set("yarn.api-service.service.client.class", - ServiceClientTest.class.getName()); - ApiServer.setServiceClient(mockServerClient); - this.apiServer = new ApiServer(conf); + private static ApiServer apiServer; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YARN_API_SERVICE_SOLR_STORAGE_ENABLED, false); + URL url = Thread.currentThread().getContextClassLoader() + .getResource("yarn-site.xml"); + if (url == null) { + throw new RuntimeException( + "Could not find 'yarn-site.xml' dummy file in classpath"); + } + conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, + new File(url.getPath()).getParent()); + //write the document to a buffer (not directly to the file, as that + //can cause the file being written to get read -which will then fail. + ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); + conf.writeXml(bytesOut); + bytesOut.close(); + //write the bytes to the file in the classpath + OutputStream os = new FileOutputStream(new File(url.getPath())); + os.write(bytesOut.toByteArray()); + os.close(); + + ServiceClient mockServiceClient = new ServiceClientBadTest(); + apiServer = new ApiServer(mockServiceClient); + + } + + @AfterClass + public static void teardownAfterClass() { + YarnSolrClientTest.cleanup(); } + /** + * Verify the path for webservice is /ws/v1. + */ @Test public void testPathAnnotation() { - assertNotNull(this.apiServer.getClass().getAnnotation(Path.class)); + assertNotNull(apiServer.getClass().getAnnotation(Path.class)); assertTrue("The controller has the annotation Path", - this.apiServer.getClass().isAnnotationPresent(Path.class)); - final Path path = this.apiServer.getClass() + apiServer.getClass().isAnnotationPresent(Path.class)); + final Path path = apiServer.getClass() .getAnnotation(Path.class); - assertEquals("The path has /ws/v1 annotation", path.value(), - "/ws/v1"); + assertEquals("The path is /ws/v1", "/ws/v1", + path.value()); } + /** + * Test get version number api. + */ @Test public void testGetVersion() { final Response actual = apiServer.getVersion(); - assertEquals("Version number is", actual.getStatus(), - Response.ok().build().getStatus()); + assertEquals("Version number is", Response.ok().build().getStatus(), + actual.getStatus()); } + /** + * Test creating invalid service. + */ @Test public void testBadCreateService() { Service service = new Service(); // Test for invalid argument final Response actual = apiServer.createService(service); - assertEquals("Create service is ", actual.getStatus(), - Response.status(Status.BAD_REQUEST).build().getStatus()); + assertEquals("Create service is ", Response.status(Status.BAD_REQUEST) + .build().getStatus(), actual.getStatus()); } + /** + * Test creating a valid service. + */ @Test public void testGoodCreateService() { Service service = new Service(); @@ -102,54 +147,104 @@ public void testGoodCreateService() { components.add(c); service.setComponents(components); final Response actual = apiServer.createService(service); - assertEquals("Create service is ", actual.getStatus(), - Response.status(Status.ACCEPTED).build().getStatus()); + assertEquals("Create service is ", + Response.status(Status.ACCEPTED).build().getStatus(), + actual.getStatus()); } + /** + * Test for getting non-existed service specification. + */ + @Test + public void testBadGetServiceSpec() { + final Response actual = apiServer.getServiceSpec("no-jenkins"); + assertEquals("Get service spec is ", + Response.status(Status.NOT_FOUND).build().getStatus(), + actual.getStatus()); + } + + /** + * Test for getting service specification. + */ + @Test + public void testGoodGetServiceSpec() { + final Response actual = apiServer.getServiceSpec("jenkins"); + assertEquals("Get service spec is ", + Response.status(Status.OK).build().getStatus(), + actual.getStatus()); + } + + /** + * Test for getting non-existed service status. + */ @Test public void testBadGetService() { - final Response actual = apiServer.getService("no-jenkins"); - assertEquals("Get service is ", actual.getStatus(), - Response.status(Status.NOT_FOUND).build().getStatus()); + final Response actual = apiServer.getServiceStatus("no-jenkins"); + assertEquals("Get service is ", + Response.status(Status.NOT_FOUND).build().getStatus(), + actual.getStatus()); } + /** + * Test for getting service with bad input. + */ @Test public void testBadGetService2() { - final Response actual = apiServer.getService(null); - assertEquals("Get service is ", actual.getStatus(), + final Response actual = apiServer.getServiceStatus(null); + assertEquals("Get service is ", Response.status(Status.INTERNAL_SERVER_ERROR) - .build().getStatus()); + .build().getStatus(), + actual.getStatus()); } + /** + * Test for getting valid service status. + */ @Test public void testGoodGetService() { - final Response actual = apiServer.getService("jenkins"); - assertEquals("Get service is ", actual.getStatus(), - Response.status(Status.OK).build().getStatus()); + final Response actual = apiServer.getServiceStatus("jenkins"); + assertEquals("Get service is ", + Response.status(Status.OK).build().getStatus(), + actual.getStatus()); } + /** + * Test for deleting non-existed service. + */ @Test public void testBadDeleteService() { final Response actual = apiServer.deleteService("no-jenkins"); - assertEquals("Delete service is ", actual.getStatus(), - Response.status(Status.BAD_REQUEST).build().getStatus()); + assertEquals("Delete service is ", + Response.status(Status.BAD_REQUEST).build().getStatus(), + actual.getStatus()); } + /** + * Test for deleting service with bad input. + */ @Test public void testBadDeleteService2() { final Response actual = apiServer.deleteService(null); - assertEquals("Delete service is ", actual.getStatus(), + assertEquals("Delete service is ", Response.status(Status.INTERNAL_SERVER_ERROR) - .build().getStatus()); + .build().getStatus(), + actual.getStatus()); } + /** + * Test for deleting service. + */ @Test public void testGoodDeleteService() { final Response actual = apiServer.deleteService("jenkins"); - assertEquals("Delete service is ", actual.getStatus(), - Response.status(Status.OK).build().getStatus()); + assertEquals("Delete service is ", + Response.status(Status.OK).build().getStatus(), + actual.getStatus()); } + /** + * Test for decrease containers in a service. + */ @Test public void testDecreaseContainerAndStop() { Service service = new Service(); @@ -170,12 +265,21 @@ public void testDecreaseContainerAndStop() { c.setResource(resource); components.add(c); service.setComponents(components); - final Response actual = apiServer.updateService("jenkins", + final Response actual = apiServer.updateServiceSpec("jenkins", + service); + assertEquals("update service spec is ", + Response.status(Status.OK).build().getStatus(), + actual.getStatus()); + final Response actual2 = apiServer.updateServiceState("jenkins", service); - assertEquals("update service is ", actual.getStatus(), - Response.status(Status.OK).build().getStatus()); + assertEquals("update service state is ", + Response.status(Status.OK).build().getStatus(), + actual2.getStatus()); } + /** + * Test for decrease containers and stop service. + */ @Test public void testBadDecreaseContainerAndStop() { Service service = new Service(); @@ -197,12 +301,21 @@ public void testBadDecreaseContainerAndStop() { components.add(c); service.setComponents(components); System.out.println("before stop"); - final Response actual = apiServer.updateService("no-jenkins", + final Response actual = apiServer.updateServiceSpec("no-jenkins", service); - assertEquals("flex service is ", actual.getStatus(), - Response.status(Status.BAD_REQUEST).build().getStatus()); + assertEquals("update service spec is ", + Response.status(Status.NOT_FOUND).build().getStatus(), + actual.getStatus()); + final Response actual2 = apiServer.updateServiceSpec("no-jenkins", + service); + assertEquals("update service state is ", + Response.status(Status.NOT_FOUND).build().getStatus(), + actual2.getStatus()); } + /** + * Test for increase containers in a service. + */ @Test public void testIncreaseContainersAndStart() { Service service = new Service(); @@ -223,12 +336,21 @@ public void testIncreaseContainersAndStart() { c.setResource(resource); components.add(c); service.setComponents(components); - final Response actual = apiServer.updateService("jenkins", + final Response actual = apiServer.updateServiceSpec("jenkins", + service); + assertEquals("update service spec is ", + Response.status(Status.OK).build().getStatus(), + actual.getStatus()); + final Response actual2 = apiServer.updateServiceSpec("jenkins", service); - assertEquals("flex service is ", actual.getStatus(), - Response.status(Status.OK).build().getStatus()); + assertEquals("update service state is ", + Response.status(Status.OK).build().getStatus(), + actual2.getStatus()); } + /** + * Test for increase containers for non-existed service. + */ @Test public void testBadStartServices() { Service service = new Service(); @@ -249,13 +371,17 @@ public void testBadStartServices() { c.setResource(resource); components.add(c); service.setComponents(components); - final Response actual = apiServer.updateService("no-jenkins", + final Response actual = apiServer.updateServiceState("no-jenkins", service); - assertEquals("start service is ", actual.getStatus(), + assertEquals("start service is ", Response.status(Status.INTERNAL_SERVER_ERROR).build() - .getStatus()); + .getStatus(), + actual.getStatus()); } + /** + * Test for start service. + */ @Test public void testGoodStartServices() { Service service = new Service(); @@ -276,12 +402,16 @@ public void testGoodStartServices() { c.setResource(resource); components.add(c); service.setComponents(components); - final Response actual = apiServer.updateService("jenkins", + final Response actual = apiServer.updateServiceState("jenkins", service); - assertEquals("start service is ", actual.getStatus(), - Response.status(Status.OK).build().getStatus()); + assertEquals("start service is ", + Response.status(Status.OK).build().getStatus(), + actual.getStatus()); } + /** + * Test for starting non-existed service. + */ @Test public void testBadStopServices() { Service service = new Service(); @@ -303,12 +433,16 @@ public void testBadStopServices() { components.add(c); service.setComponents(components); System.out.println("before stop"); - final Response actual = apiServer.updateService("no-jenkins", + final Response actual = apiServer.updateServiceState("no-jenkins", service); - assertEquals("stop service is ", actual.getStatus(), - Response.status(Status.BAD_REQUEST).build().getStatus()); + assertEquals("stop service is ", + Response.status(Status.BAD_REQUEST).build().getStatus(), + actual.getStatus()); } + /** + * Test for stopping a service. + */ @Test public void testGoodStopServices() { Service service = new Service(); @@ -330,14 +464,18 @@ public void testGoodStopServices() { components.add(c); service.setComponents(components); System.out.println("before stop"); - final Response actual = apiServer.updateService("jenkins", + final Response actual = apiServer.updateServiceState("jenkins", service); - assertEquals("stop service is ", actual.getStatus(), - Response.status(Status.OK).build().getStatus()); + assertEquals("stop service is ", + Response.status(Status.OK).build().getStatus(), + actual.getStatus()); } + /** + * Test for updating service with backend error. + */ @Test - public void testUpdateService() { + public void testBadUpdateService() { Service service = new Service(); service.setState(ServiceState.STARTED); service.setName("no-jenkins"); @@ -357,10 +495,30 @@ public void testUpdateService() { components.add(c); service.setComponents(components); System.out.println("before stop"); - final Response actual = apiServer.updateService("no-jenkins", + final Response actual = apiServer.updateServiceSpec("no-jenkins", service); - assertEquals("update service is ", actual.getStatus(), + assertEquals("update service is ", + Response.status(Status.NOT_FOUND) + .build().getStatus(), + actual.getStatus()); + final Response actual2 = apiServer.updateServiceState("no-jenkins", + service); + assertEquals("update service is ", Response.status(Status.INTERNAL_SERVER_ERROR) - .build().getStatus()); + .build().getStatus(), + actual2.getStatus()); + } + + /** + * Default configuration disables SOLR storage, list services + * REST API should return service unavailable message. + */ + @Test + public void testGetServicesList() { + final Response actual = apiServer.getServicesList(); + assertEquals("List services is disabled.", + Response.status(Status.SERVICE_UNAVAILABLE) + .build().getStatus(), + actual.getStatus()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiService.java new file mode 100644 index 0000000..8b51eef --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiService.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.service; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.webapp.ApiServer; +import org.apache.hadoop.yarn.service.client.ServiceTestUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; + +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + +import static org.junit.Assert.*; + +import static org.apache.hadoop.yarn.service.client.ServiceTestUtils.NUM_NMS; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.*; + +/** + * End to end api service integration test. + * + */ +public class TestApiService extends ServiceTestUtils { + + private static ApiServer apiServer; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YARN_API_SERVICE_SOLR_STORAGE_ENABLED, true); + setup(conf, NUM_NMS); + setupApiServer(); + } + + @AfterClass + public static void tearDownAfterClass() throws IOException { + shutdown(); + } + + /** + * Test rest api to create a YARN service and check spec. + */ + @Test(timeout = 200000) + public void testCreateServiceAndCheckSpec() { + Service exampleApp = new Service(); + exampleApp.setName("test-create-containers"); + exampleApp.addComponent(createComponent("compa", 2, "sleep 1000")); + + final Response actual = apiServer.createService(exampleApp); + assertEquals("Create service is ", + Response.status(Status.ACCEPTED).build().getStatus(), + actual.getStatus()); + final Response actual2 = apiServer.getServiceSpec(exampleApp.getName()); + assertEquals("Get service spec is ", + Response.status(Status.OK).build().getStatus(), + actual2.getStatus()); + } + + /** + * Test rest api to create a YARN service, then + * increase number of container to 2. + */ + @Test(timeout = 200000) + public void testUpdateServiceSpec() { + Service exampleApp = new Service(); + exampleApp.setName("test-increase-containers"); + exampleApp.addComponent(createComponent("compa", 1, "sleep 1000")); + final Response actual = apiServer.createService(exampleApp); + assertEquals("Create service is ", + Response.status(Status.ACCEPTED).build().getStatus(), + actual.getStatus()); + for(Component c2 : exampleApp.getComponents()) { + c2.setNumberOfContainers(2L); + } + + final Response actual2 = apiServer.updateServiceSpec(exampleApp.getName(), + exampleApp); + assertEquals("Update spec is ", + Response.status(Status.OK).build().getStatus(), + actual2.getStatus()); + + final Response actual3 = apiServer.updateServiceState(exampleApp.getName(), + exampleApp); + assertEquals("Stop service is ", + Response.status(Status.NO_CONTENT).build().getStatus(), + actual3.getStatus()); + + final Response actual4 = apiServer.updateServiceState(exampleApp.getName(), + exampleApp); + assertEquals("Start service is ", + Response.status(Status.NO_CONTENT).build().getStatus(), + actual4.getStatus()); + + final Response buffer = apiServer.getServiceSpec(exampleApp.getName()); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, + false); + + Service actual5 = (Service) buffer.getEntity(); + assertEquals("Number of container is ", 2L, + actual5.getComponents().get(0).getNumberOfContainers().longValue()); + } + + /** + * Create a copy of ApiServer and stub the ServiceClient + * with YarnSolrClientTest. This helps to test ServiceClient + * against embedded solr server. + * + * @return + * @throws Exception + */ + private static void setupApiServer() throws Exception { + setupServiceClient(); + if (apiServer == null) { + apiServer = new ApiServer(client); + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/ServiceClientBadTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/ServiceClientBadTest.java new file mode 100644 index 0000000..d796c9e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/ServiceClientBadTest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.client; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; + +/** + * A mock version of ServiceClient - This class is design + * to simulate various error conditions that will happen + * when a consumer class calls ServiceClient. If the service + * name is jenkins, the method will return proper values. + * If service name is null, methods will trigger null pointer + * exception. If service name is any other values, it will + * throw exceptions defined by corresponding method. + */ +public class ServiceClientBadTest extends ServiceClient { + + private Configuration config = new YarnConfiguration(); + + public ServiceClientBadTest() { + super(); + createYarnSolrClient(config); + } + + @Override + public Configuration getConfig() { + return config; + } + + @Override + protected void createYarnSolrClient(Configuration conf) { + ysc = new YarnSolrClientTest(conf); + } + + @Override + public ApplicationId actionCreate(Service service) { + String serviceName = service.getName(); + ServiceApiUtil.validateNameFormat(serviceName, getConfig()); + return ApplicationId.newInstance(System.currentTimeMillis(), 1); + } + + @Override + public Service getSpec(String appName) { + if (appName == null) { + throw new NullPointerException(); + } + if (appName.equals("jenkins")) { + return new Service(); + } else { + throw new IllegalArgumentException(); + } + } + + @Override + public Service getStatus(String appName) { + if (appName == null) { + throw new NullPointerException(); + } + if (appName.equals("jenkins")) { + return new Service(); + } else { + throw new IllegalArgumentException(); + } + } + + @Override + public int actionStart(String serviceName) + throws YarnException, IOException { + if (serviceName == null) { + throw new NullPointerException(); + } + if (serviceName.equals("jenkins")) { + return EXIT_SUCCESS; + } else { + throw new ApplicationNotFoundException(""); + } + } + + @Override + public int actionStop(String serviceName, boolean waitForAppStopped) + throws YarnException, IOException { + if (serviceName == null) { + throw new NullPointerException(); + } + if (serviceName.equals("jenkins")) { + return EXIT_SUCCESS; + } else { + throw new ApplicationNotFoundException(""); + } + } + + @Override + public int actionDestroy(String serviceName) { + if (serviceName == null) { + throw new NullPointerException(); + } + if (serviceName.equals("jenkins")) { + return EXIT_SUCCESS; + } else { + throw new IllegalArgumentException(); + } + } + + @Override + public Service updateSpec(String serviceName, Service service) { + if (serviceName == null) { + throw new NullPointerException(); + } + if (serviceName.equals("jenkins")) { + return service; + } else { + throw new IllegalArgumentException(); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/webapp/ApiServerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/webapp/ApiServerTest.java new file mode 100644 index 0000000..241692f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/webapp/ApiServerTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.webapp; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.service.client.ServiceClientTest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Configure ApiServer to use embedded SOLR. + * + */ +public class ApiServerTest extends ApiServer { + + private static final Logger LOG = + LoggerFactory.getLogger(ApiServer.class); + private ServiceClientTest SERVICE_CLIENT; + + /** + * Constructor used by integration test. + * + * @param conf + */ + public ApiServerTest(Configuration conf) { + setConf(conf); + try { + if (SERVICE_CLIENT==null) { + SERVICE_CLIENT = new ServiceClientTest(); + SERVICE_CLIENT.init(getConf()); + SERVICE_CLIENT.start(); + } + } catch (Exception e) { + LOG.error("Fail to initialize ServiceClient, ApiServer is unavailable."); + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt new file mode 100644 index 0000000..2c164c0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +# Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +such +that +the +their +then +there +these +they +this +to +was +will +with diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/params.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/params.json new file mode 100644 index 0000000..06114ef --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/params.json @@ -0,0 +1,20 @@ +{"params":{ + "query":{ + "defType":"edismax", + "q.alt":"*:*", + "rows":"10", + "fl":"*,score", + "":{"v":0} + }, + "facets":{ + "facet":"on", + "facet.mincount": "1", + "":{"v":0} + }, + "velocity":{ + "wt": "velocity", + "v.template":"browse", + "v.layout": "layout", + "":{"v":0} + } +}} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/protwords.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/protwords.txt new file mode 100644 index 0000000..1dfc0ab --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/schema.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/schema.xml new file mode 100644 index 0000000..90dc4b8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/schema.xml @@ -0,0 +1,116 @@ + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml new file mode 100644 index 0000000..392feec --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml @@ -0,0 +1,36 @@ + + + + 6.2.1 + + ${solr.data.dir:} + + + + + single + + + + + + + + + + \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/stopwords.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/stopwords.txt new file mode 100644 index 0000000..ae1e83e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/stopwords.txt @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/synonyms.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/synonyms.txt new file mode 100644 index 0000000..7f72128 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/synonyms.txt @@ -0,0 +1,29 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaafoo => aaabar +bbbfoo => bbbfoo bbbbar +cccfoo => cccbar cccbaz +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/yarn-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/yarn-site.xml new file mode 100644 index 0000000..8365286 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/yarn-site.xml @@ -0,0 +1,18 @@ + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index 851f73b..60fbdba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -28,6 +28,7 @@ ${project.parent.basedir} + 6.0.0 @@ -227,6 +228,36 @@ swagger-annotations + + org.apache.solr + solr-solrj + ${solr.version} + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + slf4j-api + + + + + + org.apache.solr + solr-core + ${solr.version} + test + + + + org.apache.solr + solr-test-framework + ${solr.version} + test + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index a3a9fd0..d29c23a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -33,6 +33,7 @@ import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.hadoop.security.KerberosAuthException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.util.VersionInfo; @@ -53,6 +54,7 @@ import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto; @@ -83,6 +85,7 @@ import org.apache.hadoop.yarn.service.utils.SliderUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Times; +import org.apache.solr.client.solrj.SolrServerException; import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; import org.apache.hadoop.yarn.service.exceptions.BadConfigException; import org.apache.hadoop.yarn.service.exceptions.SliderException; @@ -106,6 +109,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import javax.naming.ServiceUnavailableException; + import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*; import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_CREATE; import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_FLEX; @@ -119,7 +124,7 @@ implements SliderExitCodes, YarnServiceConstants { private static final Logger LOG = LoggerFactory.getLogger(ServiceClient.class); - private SliderFileSystem fs; + protected SliderFileSystem fs; //TODO disable retry so that client / rest API doesn't block? protected YarnClient yarnClient; // Avoid looking up applicationId from fs all the time. @@ -127,7 +132,7 @@ private RegistryOperations registryClient; private CuratorFramework curatorClient; - private YarnRPC rpc; + protected YarnRPC rpc; private static EnumSet terminatedStates = EnumSet.of(FINISHED, FAILED, KILLED); @@ -136,16 +141,26 @@ private static EnumSet preRunningStates = EnumSet.of(NEW, NEW_SAVING, SUBMITTED, ACCEPTED); + protected YarnSolrClient ysc; + public ServiceClient() { super(ServiceClient.class.getName()); } - @Override protected void serviceInit(Configuration configuration) + protected void createYarnSolrClient(Configuration conf) { + if (ysc==null) { + ysc = new YarnSolrClient(conf); + } + } + + @Override + protected void serviceInit(Configuration configuration) throws Exception { fs = new SliderFileSystem(configuration); yarnClient = YarnClient.createYarnClient(); rpc = YarnRPC.create(configuration); addService(yarnClient); + createYarnSolrClient(configuration); super.serviceInit(configuration); } @@ -190,7 +205,8 @@ public int actionBuild(Service service) } public int actionCreate(ActionCreateArgs args) - throws IOException, YarnException { + throws IOException, YarnException, SolrServerException, + ServiceUnavailableException { Service serviceDef; if (args.file != null) { serviceDef = loadAppJsonFromLocalFS(args); @@ -210,10 +226,22 @@ public int actionCreate(ActionCreateArgs args) } public ApplicationId actionCreate(Service service) - throws IOException, YarnException { + throws IOException, YarnException, SolrServerException, + ServiceUnavailableException { String serviceName = service.getName(); ServiceApiUtil.validateNameFormat(serviceName, getConfig()); ServiceApiUtil.validateAndResolveService(service, fs, getConfig()); + String userName = ""; + if (UserGroupInformation.isSecurityEnabled()) { + try { + userName = UserGroupInformation.getCurrentUser().getUserName(); + } catch(KerberosAuthException e) { + userName = null; + } + } + if (ysc.isEnabled()) { + ysc.deployApp(service, userName); + } verifyNoLiveAppInRM(serviceName, "create"); Path appDir = checkAppNotExistOnHdfs(service); @@ -429,6 +457,14 @@ public int actionDestroy(String serviceName) throws Exception { ServiceApiUtil.validateNameFormat(serviceName, getConfig()); verifyNoLiveAppInRM(serviceName, "destroy"); + String userName = ""; + if (UserGroupInformation.isSecurityEnabled()) { + userName = UserGroupInformation.getCurrentUser().getUserName(); + } + if (ysc.isEnabled()) { + ysc.deleteApp(serviceName, userName); + } + Path appDir = fs.buildClusterDirPath(serviceName); FileSystem fileSystem = fs.getFileSystem(); // remove from the appId cache @@ -507,16 +543,7 @@ private int actionHelp(String actionName, CommonArgs args) private void verifyNoLiveAppInRM(String serviceName, String action) throws IOException, YarnException { - Set types = new HashSet<>(1); - types.add(YarnServiceConstants.APP_TYPE); - Set tags = null; - if (serviceName != null) { - tags = Collections.singleton(SliderUtils.createNameTag(serviceName)); - } - GetApplicationsRequest request = GetApplicationsRequest.newInstance(); - request.setApplicationTypes(types); - request.setApplicationTags(tags); - request.setApplicationStates(liveStates); + GetApplicationsRequest request = buildApplicationsRequest(serviceName); List reports = yarnClient.getApplications(request); if (!reports.isEmpty()) { String message = ""; @@ -531,6 +558,20 @@ private void verifyNoLiveAppInRM(String serviceName, String action) } } + protected GetApplicationsRequest buildApplicationsRequest(String serviceName) { + Set types = new HashSet<>(1); + types.add(YarnServiceConstants.APP_TYPE); + Set tags = null; + if (serviceName != null) { + tags = Collections.singleton(SliderUtils.createNameTag(serviceName)); + } + GetApplicationsRequest request = GetApplicationsRequest.newInstance(); + request.setApplicationTypes(types); + request.setApplicationTags(tags); + request.setApplicationStates(liveStates); + return request; + } + private ApplicationId submitApp(Service app) throws IOException, YarnException { String serviceName = app.getName(); @@ -706,10 +747,25 @@ private boolean addAMLog4jResource(String serviceName, Configuration conf, return hasAMLog4j; } - public int actionStart(String serviceName) throws YarnException, IOException { + public int actionStart(String serviceName) throws YarnException, + IOException, SolrServerException, + ServiceUnavailableException { ServiceApiUtil.validateNameFormat(serviceName, getConfig()); Path appDir = checkAppExistOnHdfs(serviceName); - Service service = ServiceApiUtil.loadService(fs, serviceName); + String userName = null; + if (UserGroupInformation.isSecurityEnabled()) { + try { + userName = UserGroupInformation.getCurrentUser().getUserName(); + } catch(KerberosAuthException e) { + userName = null; + } + } + Service service; + if(ysc.isEnabled()) { + service = ysc.findAppEntry(serviceName, userName); + } else { + service = ServiceApiUtil.loadService(fs, serviceName); + } ServiceApiUtil.validateAndResolveService(service, fs, getConfig()); // see if it is actually running and bail out; verifyNoLiveAppInRM(serviceName, "thaw"); @@ -824,6 +880,40 @@ public ServiceState convertState(FinalApplicationStatus status) { return ServiceState.ACCEPTED; } + /** + * Fetch list of deployed services from SOLR. + * + * @return list of services + * @throws ServiceUnavailableException + * @throws IOException + * @throws SolrServerException + */ + public List getServicesList() + throws ServiceUnavailableException, IOException, + SolrServerException { + List list = ysc.listAppEntry(); + return list; + } + + /** + * Fetch service configuration from SOLR. + * + * @param serviceName - Service name + * @return Service specification + * @throws SolrServerException + * @throws ServiceUnavailableException + */ + public Service getSpec(String serviceName) throws IOException, + SolrServerException, ServiceUnavailableException, + ApplicationNotFoundException { + String userName = null; + if (UserGroupInformation.isSecurityEnabled()) { + userName = UserGroupInformation.getCurrentUser().getUserName(); + } + Service appSpec = ysc.findAppEntry(serviceName, userName); + return appSpec; + } + public Service getStatus(String serviceName) throws IOException, YarnException { ServiceApiUtil.validateNameFormat(serviceName, getConfig()); @@ -916,4 +1006,31 @@ private synchronized ApplicationId getAppId(String serviceName) cachedAppIds.put(serviceName, currentAppId); return currentAppId; } + + /** + * Invoke Solr Client to update service specification. + * + * @param serviceName - Deployed service name + * @param service - Service specification + * @return updated - service specification + * @throws IOException - when user is unauthorized + * @throws SolrServerException - when unable to connect to Solr + * @throws IllegalArgumentException - Invalid spec file + * @throws ServiceUnavailableException + */ + public Service updateSpec(String serviceName, Service service) + throws IOException, IllegalArgumentException, SolrServerException, + ServiceUnavailableException { + String userName = ""; + if (UserGroupInformation.isSecurityEnabled()) { + userName = UserGroupInformation.getCurrentUser().getUserName(); + } + Path appDir = fs.buildClusterDirPath(serviceName); + ServiceApiUtil.validateAndResolveService(service, fs, getConfig()); + if (ysc.isEnabled()) { + ysc.updateAppEntry(serviceName, userName, service); + } + persistAppDef(appDir, service); + return service; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/YarnSolrClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/YarnSolrClient.java new file mode 100644 index 0000000..a0ca14f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/YarnSolrClient.java @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.client; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; + +import javax.naming.ServiceUnavailableException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrInputDocument; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.conf.RestApiConstants; + +import static org.apache.hadoop.yarn.conf.YarnConfiguration.*; + +/** + * Yarn SOLR Storage driver for application metadata. + * + */ +public class YarnSolrClient { + + /** + * Storage driver for persist Yarnfile on Solr. + */ + private static final Log LOG = LogFactory.getLog(YarnSolrClient.class); + private String SOLR_URL; + private static final String DEFAULT_SOLR_URL = + "http://localhost:8983/solr/yarn"; + private boolean ENABLED; + private static int MAXFILTERLIMIT = 40; + + public YarnSolrClient() { + YarnConfiguration conf = new YarnConfiguration(); + setEnabled(conf.getBoolean(YARN_API_SERVICE_SOLR_STORAGE_ENABLED, + false)); + SOLR_URL = conf.get(YARN_API_SERVICE_SOLR_URL, + DEFAULT_SOLR_URL); + } + + public YarnSolrClient(Configuration conf) { + ENABLED = conf.getBoolean(YARN_API_SERVICE_SOLR_STORAGE_ENABLED, false); + SOLR_URL = conf.get(YARN_API_SERVICE_SOLR_URL, + DEFAULT_SOLR_URL); + } + + /** + * Enable YARN Solr Storage driver. + * @param enable + */ + protected void setEnabled(boolean enable) { + ENABLED = enable; + } + + /** + * Check if YARN Solr Storage is enabled. + * + * @return + */ + protected boolean getEnabled() { + return ENABLED; + } + + /** + * Get a Solr client. + * + * @return + */ + protected SolrClient getSolrClient() { + return new HttpSolrClient(SOLR_URL); + } + + /** + * Locate deployed application entry from Solr. + * + * @param appName + * @param username + * @return + * @throws ServiceUnavailableException + * @throws ApplicationNotFoundException + */ + public Service findAppEntry(String appName, String username) throws + ServiceUnavailableException, ApplicationNotFoundException { + check(); + boolean found = false; + Service entry = new Service(); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, + false); + + SolrClient solr = getSolrClient(); + SolrQuery query = new SolrQuery(); + query.setQuery("id:" + appName); + query.setFilterQueries("type_s:AppEntry"); + query.setRows(1); + + QueryResponse response; + try { + response = solr.query(query); + Iterator appList = response.getResults() + .listIterator(); + while (appList.hasNext()) { + SolrDocument d = appList.next(); + entry = mapper.readValue(d.get("yarnfile_s").toString(), + Service.class); + found = true; + } + if (!found) { + throw new ApplicationNotFoundException("Application entry is " + + "not found: " + appName); + } + } catch (SolrServerException | IOException e) { + LOG.error("Error in finding deployed application: " + appName, e); + } + return entry; + } + + /** + * Add an entry for deployed application to Solr. + * + * @param yarnApp + * @param username + * @return + * @throws ServiceUnavailableException + */ + public Service deployApp(Service yarnApp, String username) throws + ServiceUnavailableException { + check(); + SolrClient solr = getSolrClient(); + Collection docs = + new HashSet(); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, + false); + + try { + if (yarnApp!=null) { + // Register deployed application instance with AppList + SolrInputDocument request = new SolrInputDocument(); + request.addField("type_s", "AppEntry"); + request.addField("id", yarnApp.getName()); + request.addField("user_s", username); + request.addField("yarnfile_s", + mapper.writeValueAsString(yarnApp)); + docs.add(request); + } + + // Commit Solr changes. + UpdateResponse detailsResponse = solr.add(docs); + if (detailsResponse.getStatus() != 0) { + throw new IOException("Unable to store spec file."); + } + solr.commit(); + } catch (IOException | SolrServerException e) { + LOG.error("Fail to store service spec: ", e); + throw new ServiceUnavailableException("Fail to store service spec."); + } + return yarnApp; + } + + /** + * Delete an entry of deployed application in Solr. + * + * @param id + * @param username + * @throws IOException + * @throws SolrServerException + * @throws ServiceUnavailableException + */ + public void deleteApp(String id, String username) throws IOException, + SolrServerException, ServiceUnavailableException { + check(); + SolrClient solr = getSolrClient(); + try { + solr.deleteById(id); + solr.commit(); + } catch (SolrServerException | IOException e) { + LOG.error("Error in removing deployed application: "+id, e); + throw new ServiceUnavailableException("Fail to remove service spec."); + } + } + + public List listAppEntry() + throws ServiceUnavailableException { + check(); + String userName = ""; + List list = new ArrayList(); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, + false); + try { + SolrClient solr = getSolrClient(); + SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + query.setFilterQueries("type_s:AppEntry"); + if (UserGroupInformation.isSecurityEnabled()) { + userName = UserGroupInformation.getCurrentUser().getUserName(); + if (!userName.isEmpty()) { + query.addFilterQuery("user_s:" + userName); + } + } + query.setRows(MAXFILTERLIMIT); + + QueryResponse response; + response = solr.query(query); + Iterator appList = response.getResults() + .listIterator(); + while (appList.hasNext()) { + SolrDocument d = appList.next(); + Service entry = mapper.readValue(d.get("yarnfile_s").toString(), + Service.class); + list.add(entry); + } + } catch (SolrServerException | IOException e) { + LOG.error("Error in finding service for user: " + userName, e); + throw new ServiceUnavailableException("Fail to remove service spec."); + } + return list; + } + + /** + * Update spec file for deployed service. + * + * @param serviceName - Service name + * @param userName - Onwer of deployed service + * @param service - Service spec + * @return + * @throws IllegalArgumentException + * @throws IOException + * @throws SolrServerException + * @throws ServiceUnavailableException + */ + public Service updateAppEntry(String serviceName, String userName, + Service service) throws IllegalArgumentException, + ServiceUnavailableException { + check(); + boolean updated = false; + Collection docs = + new HashSet(); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, + false); + try { + SolrClient solr = getSolrClient(); + SolrQuery query = new SolrQuery(); + query.setQuery("id:" + serviceName); + query.setFilterQueries("type_s:AppEntry"); + if (!userName.isEmpty()) { + query.addFilterQuery("user_s:" + userName); + } + query.setRows(1); + QueryResponse response; + response = solr.query(query); + Iterator appList = response.getResults() + .listIterator(); + while (appList.hasNext()) { + SolrDocument d = appList.next(); + Service entry = mapper.readValue(d.get("yarnfile_s").toString(), + Service.class); + SolrInputDocument request = new SolrInputDocument(); + request.addField("type_s", "AppEntry"); + request.addField("id", entry.getName()); + request.addField("user_s", userName); + request.addField("yarnfile_s", + mapper.writeValueAsString(service)); + docs.add(request); + UpdateResponse detailsResponse = solr.add(docs); + if (detailsResponse.getStatus() != 0) { + throw new IOException("Unable to store spec file."); + } + solr.commit(); + updated = true; + } + } catch(IOException | SolrServerException e) { + LOG.error("Failed to update service spec for user: " + userName, e); + throw new ServiceUnavailableException("Fail to update service spec."); + } + if (!updated) { + throw new IllegalArgumentException("Service is not found."); + } + return service; + } + + /** + * Check if YARN Solr Storage feature is enabled. + * + * @return + */ + public boolean isEnabled() { + return ENABLED; + } + + /** + * Check if YARN Solr Integration is enabled. + * @throws ServiceUnavailableException + */ + public void check() throws ServiceUnavailableException { + if (!ENABLED) { + throw new ServiceUnavailableException("YARN Solr " + + "support is unavailable."); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java index 6de2dc0..eb01772 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java @@ -24,6 +24,8 @@ String VERSION = "/services/version"; String SERVICE_ROOT_PATH = "/services"; String SERVICE_PATH = "/services/{service_name}"; + String SERVICE_SPEC_PATH = "/services/{service_name}/spec"; + String SERVICE_STATUS_PATH = "/services/{service_name}/status"; String COMPONENT_PATH = "/services/{service_name}/components/{component_name}"; // Query param @@ -40,4 +42,5 @@ Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002; Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003; Integer ERROR_CODE_APP_NAME_INVALID = 404004; + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java deleted file mode 100644 index aeb4341..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service; - -import org.apache.commons.io.FileUtils; -import org.apache.curator.test.TestingCluster; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.yarn.service.api.records.Service; -import org.apache.hadoop.yarn.service.conf.YarnServiceConf; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.MiniYARNCluster; -import org.apache.hadoop.yarn.service.api.records.Component; -import org.apache.hadoop.yarn.service.api.records.Resource; -import org.apache.hadoop.yarn.service.utils.JsonSerDeser; -import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; -import org.apache.hadoop.yarn.service.utils.SliderFileSystem; -import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin; -import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree; -import org.codehaus.jackson.map.PropertyNamingStrategy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.net.URL; - -import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_PMEM_CHECK_ENABLED; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_VMEM_CHECK_ENABLED; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_SERVICE_ENABLED; -import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.AM_RESOURCE_MEM; -import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; -import static org.mockito.Matchers.anyObject; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class ServiceTestUtils { - - private static final Logger LOG = - LoggerFactory.getLogger(ServiceTestUtils.class); - - private MiniYARNCluster yarnCluster = null; - private MiniDFSCluster hdfsCluster = null; - private FileSystem fs = null; - private Configuration conf = null; - public static final int NUM_NMS = 1; - private File basedir; - - public static final JsonSerDeser JSON_SER_DESER = - new JsonSerDeser<>(Service.class, - PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); - - // Example service definition - // 2 components, each of which has 2 containers. - protected Service createExampleApplication() { - Service exampleApp = new Service(); - exampleApp.setName("example-app"); - exampleApp.addComponent(createComponent("compa")); - exampleApp.addComponent(createComponent("compb")); - return exampleApp; - } - - protected Component createComponent(String name) { - return createComponent(name, 2L, "sleep 1000"); - } - - protected Component createComponent(String name, long numContainers, - String command) { - Component comp1 = new Component(); - comp1.setNumberOfContainers(numContainers); - comp1.setLaunchCommand(command); - comp1.setName(name); - Resource resource = new Resource(); - comp1.setResource(resource); - resource.setMemory("128"); - resource.setCpus(1); - return comp1; - } - - public static SliderFileSystem initMockFs() throws IOException { - return initMockFs(null); - } - - public static SliderFileSystem initMockFs(Service ext) throws IOException { - SliderFileSystem sfs = mock(SliderFileSystem.class); - FileSystem mockFs = mock(FileSystem.class); - JsonSerDeser jsonSerDeser = mock(JsonSerDeser.class); - when(sfs.getFileSystem()).thenReturn(mockFs); - when(sfs.buildClusterDirPath(anyObject())).thenReturn( - new Path("cluster_dir_path")); - if (ext != null) { - when(jsonSerDeser.load(anyObject(), anyObject())).thenReturn(ext); - } - ServiceApiUtil.setJsonSerDeser(jsonSerDeser); - return sfs; - } - - protected void setConf(YarnConfiguration conf) { - this.conf = conf; - } - - protected Configuration getConf() { - return conf; - } - - protected FileSystem getFS() { - return fs; - } - - protected void setupInternal(int numNodeManager) - throws Exception { - LOG.info("Starting up YARN cluster"); -// Logger rootLogger = LogManager.getRootLogger(); -// rootLogger.setLevel(Level.DEBUG); - setConf(new YarnConfiguration()); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); - // reduce the teardown waiting time - conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000); - conf.set("yarn.log.dir", "target"); - // mark if we need to launch the v1 timeline server - // disable aux-service based timeline aggregators - conf.set(YarnConfiguration.NM_AUX_SERVICES, ""); - conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8"); - // Enable ContainersMonitorImpl - conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR, - LinuxResourceCalculatorPlugin.class.getName()); - conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, - ProcfsBasedProcessTree.class.getName()); - conf.setBoolean( - YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, true); - conf.setBoolean(TIMELINE_SERVICE_ENABLED, false); - conf.setInt(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100); - conf.setLong(DEBUG_NM_DELETE_DELAY_SEC, 60000); - conf.setLong(AM_RESOURCE_MEM, 526); - conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 5); - // Disable vmem check to disallow NM killing the container - conf.setBoolean(NM_VMEM_CHECK_ENABLED, false); - conf.setBoolean(NM_PMEM_CHECK_ENABLED, false); - // setup zk cluster - TestingCluster zkCluster; - zkCluster = new TestingCluster(1); - zkCluster.start(); - conf.set(YarnConfiguration.RM_ZK_ADDRESS, zkCluster.getConnectString()); - conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString()); - LOG.info("ZK cluster: " + zkCluster.getConnectString()); - - fs = FileSystem.get(conf); - basedir = new File("target", "apps"); - if (basedir.exists()) { - FileUtils.deleteDirectory(basedir); - } else { - basedir.mkdirs(); - } - - conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath()); - - if (yarnCluster == null) { - yarnCluster = - new MiniYARNCluster(TestYarnNativeServices.class.getSimpleName(), 1, - numNodeManager, 1, 1); - yarnCluster.init(conf); - yarnCluster.start(); - - waitForNMsToRegister(); - - URL url = Thread.currentThread().getContextClassLoader() - .getResource("yarn-site.xml"); - if (url == null) { - throw new RuntimeException( - "Could not find 'yarn-site.xml' dummy file in classpath"); - } - Configuration yarnClusterConfig = yarnCluster.getConfig(); - yarnClusterConfig.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, - new File(url.getPath()).getParent()); - //write the document to a buffer (not directly to the file, as that - //can cause the file being written to get read -which will then fail. - ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); - yarnClusterConfig.writeXml(bytesOut); - bytesOut.close(); - //write the bytes to the file in the classpath - OutputStream os = new FileOutputStream(new File(url.getPath())); - os.write(bytesOut.toByteArray()); - os.close(); - LOG.info("Write yarn-site.xml configs to: " + url); - } - if (hdfsCluster == null) { - HdfsConfiguration hdfsConfig = new HdfsConfiguration(); - hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig) - .numDataNodes(1).build(); - } - - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - LOG.info("setup thread sleep interrupted. message=" + e.getMessage()); - } - } - - public void shutdown() throws IOException { - if (yarnCluster != null) { - try { - yarnCluster.stop(); - } finally { - yarnCluster = null; - } - } - if (hdfsCluster != null) { - try { - hdfsCluster.shutdown(); - } finally { - hdfsCluster = null; - } - } - if (basedir != null) { - FileUtils.deleteDirectory(basedir); - } - SliderFileSystem sfs = new SliderFileSystem(conf); - Path appDir = sfs.getBaseApplicationPath(); - sfs.getFileSystem().delete(appDir, true); - } - - private void waitForNMsToRegister() throws Exception { - int sec = 60; - while (sec >= 0) { - if (yarnCluster.getResourceManager().getRMContext().getRMNodes().size() - >= NUM_NMS) { - break; - } - Thread.sleep(1000); - sec--; - } - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java index 959e4d6..7f10a25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java @@ -21,6 +21,7 @@ import org.apache.hadoop.registry.client.api.RegistryConstants; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.client.ServiceTestUtils; import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages; import org.apache.hadoop.yarn.service.api.records.Artifact; import org.apache.hadoop.yarn.service.api.records.Component; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java index d821b84..9cb710f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java @@ -18,23 +18,24 @@ package org.apache.hadoop.yarn.service; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Container; import org.apache.hadoop.yarn.service.api.records.ContainerState; import org.apache.hadoop.yarn.service.client.ServiceClient; -import org.apache.hadoop.yarn.service.exceptions.SliderException; +import org.apache.hadoop.yarn.service.client.ServiceTestUtils; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; -import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -52,6 +53,7 @@ import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED; import static org.apache.hadoop.yarn.conf.YarnConfiguration.*; +import static org.junit.Assert.*; /** * End to end tests to test deploying services with MiniYarnCluster and a in-JVM @@ -65,26 +67,31 @@ @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); - @Before - public void setup() throws Exception { - setupInternal(NUM_NMS); + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YARN_API_SERVICE_SOLR_STORAGE_ENABLED, true); + setup(conf, NUM_NMS); + setupServiceClient(); } - @After - public void tearDown() throws IOException { + @AfterClass + public static void tearDownAfterClass() throws IOException { shutdown(); } // End-to-end test to use ServiceClient to deploy a service. // 1. Create a service with 2 components, each of which has 2 containers - // 2. Flex up each component to 3 containers and check the component instance names - // 3. Flex down each component to 1 container and check the component instance names - // 4. Flex up each component to 2 containers and check the component instance names + // 2. Flex up each component to 3 containers and check the component + // instance names + // 3. Flex down each component to 1 container and check the component + // instance names + // 4. Flex up each component to 2 containers and check the component + // instance names // 5. Stop the service // 6. Destroy the service @Test (timeout = 200000) public void testCreateFlexStopDestroyService() throws Exception { - ServiceClient client = createClient(); Service exampleApp = createExampleApplication(); client.actionCreate(exampleApp); SliderFileSystem fileSystem = new SliderFileSystem(getConf()); @@ -135,7 +142,6 @@ public void testCreateFlexStopDestroyService() throws Exception { // Check containers for compa started before containers for compb @Test (timeout = 200000) public void testComponentStartOrder() throws Exception { - ServiceClient client = createClient(); Service exampleApp = new Service(); exampleApp.setName("teststartorder"); exampleApp.addComponent(createComponent("compa", 2, "sleep 1000")); @@ -185,13 +191,39 @@ private void checkContainerLaunchDependencies(ServiceClient client, } } + /** + * Test create a service, change number of containers, restart service. + * @throws Exception + */ + @Test (timeout = 200000) + public void testChangeSpec() throws Exception { + Service exampleApp = new Service(); + exampleApp.setName("testincreasecontainers"); + exampleApp.addComponent(createComponent("compa", 2, "sleep 1000")); + client.actionCreate(exampleApp); + waitForAllCompToBeReady(client, exampleApp); + + List list = new ArrayList(); + list.add(createComponent("compa", 3, "sleep 1000")); + exampleApp.setComponents(list); + client.updateSpec(exampleApp.getName(), exampleApp); + client.actionStop(exampleApp.getName(), true); + client.actionStart(exampleApp.getName()); + Service actual = client.getSpec(exampleApp.getName()); + assertEquals("Number of containers should be: ", + exampleApp.getComponents().get(0).getNumberOfContainers().longValue(), + actual.getComponents().get(0).getNumberOfContainers().longValue()); + client.actionStop(exampleApp.getName(), true); + client.actionDestroy(exampleApp.getName()); + } private Map flexComponents(ServiceClient client, Service exampleApp, long count) throws YarnException, IOException { Map compCounts = new HashMap<>(); compCounts.put("compa", count); compCounts.put("compb", count); - // flex will update the persisted conf to reflect latest number of containers. + // flex will update the persisted conf to reflect latest number of + // containers. exampleApp.getComponent("compa").setNumberOfContainers(count); exampleApp.getComponent("compb").setNumberOfContainers(count); client.flexByRestService(exampleApp.getName(), compCounts); @@ -199,9 +231,11 @@ private void checkContainerLaunchDependencies(ServiceClient client, } // Check each component's comp instances name are in sequential order. - // E.g. If there are two instances compA-1 and compA-2 - // When flex up to 4 instances, it should be compA-1 , compA-2, compA-3, compA-4 - // When flex down to 3 instances, it should be compA-1 , compA-2, compA-3. + // E.g. If there are two instances compA-1 and compA-2. + // When flex up to 4 instances, it should be compA-1 , compA-2, + // compA-3, compA-4. + // When flex down to 3 instances, it should be compA-1 , + // compA-2, compA-3. private void checkCompInstancesInOrder(ServiceClient client, Service exampleApp) throws IOException, YarnException { Service service = client.getStatus(exampleApp.getName()); @@ -297,21 +331,6 @@ private void waitForAllCompToBeReady(ServiceClient client, }, 2000, 200000); } - private ServiceClient createClient() throws Exception { - ServiceClient client = new ServiceClient() { - @Override protected Path addJarResource(String appName, - Map localResources) - throws IOException, SliderException { - // do nothing, the Unit test will use local jars - return null; - } - }; - client.init(getConf()); - client.start(); - return client; - } - - private int countTotalContainers(Service service) { int totalContainers = 0; for (Component component : service.getComponents()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/EmbeddedSolrServerFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/EmbeddedSolrServerFactory.java new file mode 100644 index 0000000..db6d2f9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/EmbeddedSolrServerFactory.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.service.client; + +import org.apache.commons.io.FileUtils; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.core.NodeConfig; +import org.apache.solr.core.SolrResourceLoader; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Unit test utility class for testing SOLR. + */ +public final class EmbeddedSolrServerFactory { + + private EmbeddedSolrServerFactory() { + } + + /** + * Cleans the given solrHome directory and creates a new + * EmbeddedSolrServer. + * + * @param solrHome + * the Solr home directory to use + * @param configSetHome + * the directory containing config sets + * @param coreName + * the name of the core, must have a matching directory in + * configHome + * + * @return an EmbeddedSolrServer with a core created for the given + * coreName + * @throws IOException + */ + public static SolrClient create(final String solrHome, + final String configSetHome, final String coreName) + throws IOException, SolrServerException { + return create(solrHome, configSetHome, coreName, true); + } + + /** + * @param solrHome + * the Solr home directory to use + * @param configSetHome + * the directory containing config sets + * @param coreName + * the name of the core, must have a matching directory in + * configHome + * @param cleanSolrHome + * if true the directory for solrHome will be deleted and + * re-created if it already exists + * + * @return an EmbeddedSolrServer with a core created for the given + * coreName + * @throws IOException + */ + public static SolrClient create(final String solrHome, + final String configSetHome, final String coreName, + final boolean cleanSolrHome) throws IOException, + SolrServerException { + + final File solrHomeDir = new File(solrHome); + if (solrHomeDir.exists() && cleanSolrHome) { + FileUtils.deleteDirectory(solrHomeDir); + solrHomeDir.mkdirs(); + } else { + solrHomeDir.mkdirs(); + } + + final SolrResourceLoader loader = new SolrResourceLoader( + solrHomeDir.toPath()); + final Path configSetPath = Paths + .get(configSetHome).toAbsolutePath(); + + final NodeConfig config = new NodeConfig.NodeConfigBuilder( + "embeddedSolrServerNode", loader) + .setConfigSetBaseDirectory(configSetPath + .toString()).build(); + + final EmbeddedSolrServer embeddedSolrServer = + new EmbeddedSolrServer(config, coreName); + + final CoreAdminRequest.Create createRequest = + new CoreAdminRequest.Create(); + createRequest.setCoreName(coreName); + createRequest.setConfigSet(coreName); + embeddedSolrServer.request(createRequest); + + return embeddedSolrServer; + } + +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/ServiceClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/ServiceClientTest.java new file mode 100644 index 0000000..ff56d34 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/ServiceClientTest.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.service.client; + +import java.io.IOException; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.service.exceptions.SliderException; +import org.apache.hadoop.yarn.service.utils.SliderFileSystem; + +/** + * Unit test setup class for ServiceClient. This class triggers + * unit test to use embedded solr. + * + */ +public class ServiceClientTest extends ServiceClient { + + @Override + protected void createYarnSolrClient(Configuration conf) { + if (ysc==null) { + ysc = new YarnSolrClientTest(conf); + } + } + + @Override + protected Path addJarResource(String appName, + Map localResources) + throws IOException, SliderException { + // do nothing, the Unit test will use local jars + return null; + } + + @Override + protected void serviceInit(Configuration configuration) + throws Exception { + fs = new SliderFileSystem(configuration); + yarnClient = YarnClient.createYarnClient(); + rpc = YarnRPC.create(configuration); + addService(yarnClient); + createYarnSolrClient(configuration); + super.serviceInit(configuration); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/ServiceTestUtils.java new file mode 100644 index 0000000..4080e12 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/ServiceTestUtils.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.client; + +import org.apache.commons.io.FileUtils; +import org.apache.curator.test.TestingCluster; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.client.ServiceClient; +import org.apache.hadoop.yarn.service.conf.YarnServiceConf; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.MiniYARNCluster; +import org.apache.hadoop.yarn.service.TestYarnNativeServices; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.Resource; +import org.apache.hadoop.yarn.service.utils.JsonSerDeser; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; +import org.apache.hadoop.yarn.service.utils.SliderFileSystem; +import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin; +import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree; +import org.codehaus.jackson.map.PropertyNamingStrategy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URL; + +import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_PMEM_CHECK_ENABLED; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_VMEM_CHECK_ENABLED; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_SERVICE_ENABLED; +import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.AM_RESOURCE_MEM; +import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; +import static org.mockito.Matchers.anyObject; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Utility class for setting up mini cluster for YARN service test. + * + */ +public class ServiceTestUtils { + + private static final Logger LOG = + LoggerFactory.getLogger(ServiceTestUtils.class); + + private static MiniYARNCluster yarnCluster = null; + private static MiniDFSCluster hdfsCluster = null; + private static FileSystem fs = null; + private static Configuration conf = null; + public static final int NUM_NMS = 1; + private static File basedir; + protected static ServiceClient client = null; + + public static final JsonSerDeser JSON_SER_DESER = + new JsonSerDeser<>(Service.class, + PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); + + // Example service definition + // 2 components, each of which has 2 containers. + protected Service createExampleApplication() { + Service exampleApp = new Service(); + exampleApp.setName("example-app"); + exampleApp.addComponent(createComponent("compa")); + exampleApp.addComponent(createComponent("compb")); + return exampleApp; + } + + protected Component createComponent(String name) { + return createComponent(name, 2L, "sleep 1000"); + } + + protected Component createComponent(String name, long numContainers, + String command) { + Component comp1 = new Component(); + comp1.setNumberOfContainers(numContainers); + comp1.setLaunchCommand(command); + comp1.setName(name); + Resource resource = new Resource(); + comp1.setResource(resource); + resource.setMemory("128"); + resource.setCpus(1); + return comp1; + } + + public static SliderFileSystem initMockFs() throws IOException { + return initMockFs(null); + } + + public static SliderFileSystem initMockFs(Service ext) throws IOException { + SliderFileSystem sfs = mock(SliderFileSystem.class); + FileSystem mockFs = mock(FileSystem.class); + JsonSerDeser jsonSerDeser = mock(JsonSerDeser.class); + when(sfs.getFileSystem()).thenReturn(mockFs); + when(sfs.buildClusterDirPath(anyObject())).thenReturn( + new Path("cluster_dir_path")); + if (ext != null) { + when(jsonSerDeser.load(anyObject(), anyObject())).thenReturn(ext); + } + ServiceApiUtil.setJsonSerDeser(jsonSerDeser); + return sfs; + } + + protected static void setConf(Configuration config) { + conf = config; + } + + protected static Configuration getConf() { + return conf; + } + + protected FileSystem getFS() { + return fs; + } + + protected static void setup(Configuration testConf, + int numNodeManager) + throws Exception { + LOG.info("Starting up YARN cluster"); + setConf(testConf); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); + // reduce the teardown waiting time + conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000); + conf.set("yarn.log.dir", "target"); + // mark if we need to launch the v1 timeline server + // disable aux-service based timeline aggregators + conf.set(YarnConfiguration.NM_AUX_SERVICES, ""); + conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8"); + // Enable ContainersMonitorImpl + conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR, + LinuxResourceCalculatorPlugin.class.getName()); + conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, + ProcfsBasedProcessTree.class.getName()); + conf.setBoolean( + YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, true); + conf.setBoolean(TIMELINE_SERVICE_ENABLED, false); + conf.setInt(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100); + conf.setLong(DEBUG_NM_DELETE_DELAY_SEC, 60000); + conf.setLong(AM_RESOURCE_MEM, 526); + conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 5); + // Disable vmem check to disallow NM killing the container + conf.setBoolean(NM_VMEM_CHECK_ENABLED, false); + conf.setBoolean(NM_PMEM_CHECK_ENABLED, false); + // setup zk cluster + TestingCluster zkCluster; + zkCluster = new TestingCluster(1); + zkCluster.start(); + conf.set(YarnConfiguration.RM_ZK_ADDRESS, zkCluster.getConnectString()); + conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString()); + LOG.info("ZK cluster: " + zkCluster.getConnectString()); + + fs = FileSystem.get(conf); + basedir = new File("target", "apps"); + if (basedir.exists()) { + FileUtils.deleteDirectory(basedir); + } else { + basedir.mkdirs(); + } + + conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath()); + + if (yarnCluster == null) { + yarnCluster = + new MiniYARNCluster(TestYarnNativeServices.class.getSimpleName(), 1, + numNodeManager, 1, 1); + yarnCluster.init(conf); + yarnCluster.start(); + + waitForNMsToRegister(); + + URL url = Thread.currentThread().getContextClassLoader() + .getResource("yarn-site.xml"); + if (url == null) { + throw new RuntimeException( + "Could not find 'yarn-site.xml' dummy file in classpath"); + } + Configuration yarnClusterConfig = yarnCluster.getConfig(); + yarnClusterConfig.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, + new File(url.getPath()).getParent()); + //write the document to a buffer (not directly to the file, as that + //can cause the file being written to get read -which will then fail. + ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); + yarnClusterConfig.writeXml(bytesOut); + bytesOut.close(); + //write the bytes to the file in the classpath + OutputStream os = new FileOutputStream(new File(url.getPath())); + os.write(bytesOut.toByteArray()); + os.close(); + LOG.info("Write yarn-site.xml configs to: " + url); + } + if (hdfsCluster == null) { + HdfsConfiguration hdfsConfig = new HdfsConfiguration(); + hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig) + .numDataNodes(1).build(); + } + + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + LOG.info("setup thread sleep interrupted. message=" + e.getMessage()); + } + } + + public static void shutdown() throws IOException { + if (yarnCluster != null) { + try { + yarnCluster.stop(); + } finally { + yarnCluster = null; + } + } + if (hdfsCluster != null) { + try { + hdfsCluster.shutdown(); + } finally { + hdfsCluster = null; + } + } + if (basedir != null) { + FileUtils.deleteDirectory(basedir); + } + SliderFileSystem sfs = new SliderFileSystem(conf); + Path appDir = sfs.getBaseApplicationPath(); + sfs.getFileSystem().delete(appDir, true); + } + + private static void waitForNMsToRegister() throws Exception { + int sec = 60; + while (sec >= 0) { + if (yarnCluster.getResourceManager().getRMContext().getRMNodes().size() + >= NUM_NMS) { + break; + } + Thread.sleep(1000); + sec--; + } + } + + public static void setupServiceClient() throws Exception { + if (client==null) { + client = new ServiceClientTest(); + client.init(getConf()); + client.start(); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/YarnSolrClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/YarnSolrClientTest.java new file mode 100644 index 0000000..0e0665c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/YarnSolrClientTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.client; + +import java.io.File; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.*; + +/** + * Mocked version of Yarn Solr Client for testing with + * embedded SOLR server. + * + */ +public class YarnSolrClientTest extends YarnSolrClient { + + private static final Logger LOG = + LoggerFactory.getLogger(ServiceTestUtils.class); + + static final String CONFIGSET_DIR = "src/test/resources/configsets"; + private static String targetLocation = "target"; + private static String solrHome = targetLocation + "/solr"; + private static SolrClient solr; + + public YarnSolrClientTest(Configuration conf) { + LOG.info("solrHome directory: " + solrHome); + setEnabled(conf.getBoolean(YARN_API_SERVICE_SOLR_STORAGE_ENABLED, + false)); + LOG.info("YARN Solr Client enabled: " + getEnabled()); + if (getEnabled()) { + try { + solr = EmbeddedSolrServerFactory.create(solrHome, + CONFIGSET_DIR, "exampleCollection", true); + } catch (IOException | SolrServerException e) { + LOG.warn("Fail to enable Embedded Solr Server," + + "YARN Solr Storage is disabled."); + setEnabled(false); + } + } + } + + @Override + protected SolrClient getSolrClient() { + return solr; + } + + public static void cleanup() { + File f = new File(solrHome); + if (f.exists()) { + f.delete(); + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java index 5fdd2ab..a4df02c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java @@ -21,12 +21,12 @@ import org.apache.hadoop.yarn.service.api.records.Service; +import static org.apache.hadoop.yarn.service.client.ServiceTestUtils.JSON_SER_DESER; + import java.io.IOException; import java.util.ArrayList; import java.util.List; -import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER; - /** * Names of the example configs. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java index 8739382..98cebba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java @@ -19,8 +19,8 @@ package org.apache.hadoop.yarn.service.conf; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.service.ServiceTestUtils; import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.client.ServiceTestUtils; import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.hadoop.yarn.service.api.records.Configuration; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java index a813da3..2085a2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java @@ -19,8 +19,8 @@ package org.apache.hadoop.yarn.service.conf; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.service.ServiceTestUtils; import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.client.ServiceTestUtils; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; import org.junit.Assert; @@ -28,11 +28,11 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import static org.apache.hadoop.yarn.service.client.ServiceTestUtils.JSON_SER_DESER; + import java.util.Arrays; import java.util.Collection; -import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER; - /** * Test loading example resources. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java index 0e03a2c..44f43b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java @@ -23,9 +23,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.service.MockServiceAM; -import org.apache.hadoop.yarn.service.ServiceTestUtils; - import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.client.ServiceTestUtils; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.conf.YarnServiceConf; import org.junit.After; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt new file mode 100644 index 0000000..2c164c0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +# Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +such +that +the +their +then +there +these +they +this +to +was +will +with diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/params.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/params.json new file mode 100644 index 0000000..06114ef --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/params.json @@ -0,0 +1,20 @@ +{"params":{ + "query":{ + "defType":"edismax", + "q.alt":"*:*", + "rows":"10", + "fl":"*,score", + "":{"v":0} + }, + "facets":{ + "facet":"on", + "facet.mincount": "1", + "":{"v":0} + }, + "velocity":{ + "wt": "velocity", + "v.template":"browse", + "v.layout": "layout", + "":{"v":0} + } +}} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/protwords.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/protwords.txt new file mode 100644 index 0000000..1dfc0ab --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/schema.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/schema.xml new file mode 100644 index 0000000..90dc4b8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/schema.xml @@ -0,0 +1,116 @@ + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml new file mode 100644 index 0000000..392feec --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml @@ -0,0 +1,36 @@ + + + + 6.2.1 + + ${solr.data.dir:} + + + + + single + + + + + + + + + + \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/stopwords.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/stopwords.txt new file mode 100644 index 0000000..ae1e83e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/stopwords.txt @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/synonyms.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/synonyms.txt new file mode 100644 index 0000000..7f72128 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/synonyms.txt @@ -0,0 +1,29 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaafoo => aaabar +bbbfoo => bbbfoo bbbbar +cccfoo => cccbar cccbaz +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/yarn-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/yarn-site.xml index 266caa9..a871c6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/yarn-site.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/yarn-site.xml @@ -16,4 +16,8 @@ + + yarn.api-service.solr.storage.enabled + true + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 132d992..61b1c36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3478,5 +3478,20 @@ auto + + yarn.api-service.solr.storage.enabled + false + + Flag to enable YARN Services for storing service configuration + on solr. + + + + yarn.api-service.solr.url + http://localhost:8983/solr/yarn + + URL to Solr server for storing YARN Service config. + +