commit 16e835f029614cba8d45f1bcdbe0429a2430336e Author: eyang Date: Wed Oct 4 13:17:01 2017 -0400 YARN-7216. Added Solr as storage for YARN service spec files. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index a3b1db1..6df6c2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -124,11 +124,37 @@ public Response createService(Service service) { } @GET - @Path(SERVICE_PATH) + @Path(SERVICE_SPEC_PATH) + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response getServiceSpec(@PathParam(SERVICE_NAME) + String appName) { + LOG.info("GET: getServiceConfig for appName = {}", appName); + ServiceStatus serviceStatus = new ServiceStatus(); + try { + Service app = SERVICE_CLIENT.getSpec(appName); + return Response.ok(app).build(); + } catch (IllegalArgumentException e) { + serviceStatus.setDiagnostics(e.getMessage()); + serviceStatus.setCode(ERROR_CODE_APP_NAME_INVALID); + return Response.status(Status.NOT_FOUND).entity(serviceStatus) + .build(); + } catch (Exception e) { + LOG.error("Get service failed", e); + serviceStatus + .setDiagnostics("Failed to retrieve service: " + e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(serviceStatus).build(); + } + } + + @GET + @Path(SERVICE_STATUS_PATH) @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) - public Response getService(@PathParam(SERVICE_NAME) String appName) { - LOG.info("GET: getService for appName = {}", appName); + public Response getServiceStatus(@PathParam(SERVICE_NAME) + String appName) { + LOG.info("GET: getServiceStatus for appName = {}", appName); ServiceStatus serviceStatus = new ServiceStatus(); try { Service app = SERVICE_CLIENT.getStatus(appName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java index fc65a63..f4acd94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java @@ -74,7 +74,7 @@ public ApiServerWebApp() { @Override protected void serviceStart() throws Exception { bindAddress = getConfig().getSocketAddr(API_SERVER_ADDRESS, - DEFAULT_API_SERVER_ADDRESS , DEFAULT_API_SERVER_PORT); + DEFAULT_API_SERVER_ADDRESS, DEFAULT_API_SERVER_PORT); logger.info("YARN API server running on " + bindAddress); if (UserGroupInformation.isSecurityEnabled()) { doSecureLogin(getConfig()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java index 3e08c3a..f74bcd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java @@ -56,6 +56,18 @@ public ApplicationId actionCreate(Service service) { } @Override + public Service getSpec(String appName) { + if (appName == null) { + throw new NullPointerException(); + } + if (appName.equals("jenkins")) { + return new Service(); + } else { + throw new IllegalArgumentException(); + } + } + + @Override public Service getStatus(String appName) { if (appName == null) { throw new NullPointerException(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java index 1c352b8..34af930 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ServiceState; import org.apache.hadoop.yarn.service.client.ServiceClient; +import org.apache.hadoop.yarn.service.conf.RestApiConstants; import org.apache.hadoop.yarn.service.webapp.ApiServer; import javax.ws.rs.Path; import javax.ws.rs.core.Response; @@ -53,6 +54,7 @@ public void setup() throws Exception { Configuration conf = new Configuration(); conf.set("yarn.api-service.service.client.class", ServiceClientTest.class.getName()); + conf.setBoolean(RestApiConstants.YARN_SOLR_STORAGE_ENABLED, true); ApiServer.setApiServer(mockServerClient); this.apiServer = new ApiServer(conf); } @@ -109,15 +111,30 @@ public void testGoodCreateService() { } @Test + public void testBadGetServiceSpec() { + final Response actual = apiServer.getServiceSpec("no-jenkins"); + assertThat("Get service spec is ", actual.getStatus(), + is(Response.status(Status.NOT_FOUND).build().getStatus())); + } + + @Test + public void testGoodGetServiceSpec() { + final Response actual = apiServer.getServiceSpec("jenkins"); + assertThat("Get service spec is ", actual.getStatus(), + is(Response.status(Status.OK) + .build().getStatus())); + } + + @Test public void testBadGetService() { - final Response actual = apiServer.getService("no-jenkins"); + final Response actual = apiServer.getServiceStatus("no-jenkins"); assertThat("Get service is ", actual.getStatus(), is(Response.status(Status.NOT_FOUND).build().getStatus())); } @Test public void testBadGetService2() { - final Response actual = apiServer.getService(null); + final Response actual = apiServer.getServiceStatus(null); assertThat("Get service is ", actual.getStatus(), is(Response.status(Status.INTERNAL_SERVER_ERROR) .build().getStatus())); @@ -125,7 +142,7 @@ public void testBadGetService2() { @Test public void testGoodGetService() { - final Response actual = apiServer.getService("jenkins"); + final Response actual = apiServer.getServiceStatus("jenkins"); assertThat("Get service is ", actual.getStatus(), is(Response.status(Status.OK).build().getStatus())); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiService.java index 2554e8f..f4d96d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiService.java @@ -63,10 +63,10 @@ public void tearDown() throws IOException { } /** - * Test rest api to create a YARN service. + * Test rest api to create a YARN service and check spec. */ @Test(timeout = 200000) - public void testCreateService() { + public void testCreateServiceAndCheckSpec() { Service service = new Service(); service.setName("jenkins"); Artifact artifact = new Artifact(); @@ -87,6 +87,9 @@ public void testCreateService() { final Response actual = apiServer.createService(service); assertThat("Create service is ", actual.getStatus(), is(Response.status(Status.ACCEPTED).build().getStatus())); + final Response actual2 = apiServer.getServiceSpec("jenkins"); + assertThat("Get service spec is ", actual2.getStatus(), + is(Response.status(Status.OK).build().getStatus())); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/yarn-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/yarn-site.xml index 266caa9..e906a3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/yarn-site.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/yarn-site.xml @@ -15,5 +15,6 @@ --> - + yarn.api-service.solr.storage.enabled + true diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index 205a64d..f2bccb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -28,6 +28,7 @@ ${project.parent.basedir} + 6.3.0 @@ -222,6 +223,36 @@ swagger-annotations + + org.apache.solr + solr-solrj + ${solr.version} + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + slf4j-api + + + + + + org.apache.solr + solr-core + ${solr.version} + test + + + + org.apache.solr + solr-test-framework + ${solr.version} + test + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index a3a9fd0..9edb1e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -33,6 +33,7 @@ import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.hadoop.security.KerberosAuthException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.util.VersionInfo; @@ -136,16 +137,24 @@ private static EnumSet preRunningStates = EnumSet.of(NEW, NEW_SAVING, SUBMITTED, ACCEPTED); + protected YarnSolrClient ysc; + public ServiceClient() { super(ServiceClient.class.getName()); } - @Override protected void serviceInit(Configuration configuration) + protected void createYarnSolrClient() { + ysc = new YarnSolrClient(); + } + + @Override + protected void serviceInit(Configuration configuration) throws Exception { fs = new SliderFileSystem(configuration); yarnClient = YarnClient.createYarnClient(); rpc = YarnRPC.create(configuration); addService(yarnClient); + createYarnSolrClient(); super.serviceInit(configuration); } @@ -214,6 +223,15 @@ public ApplicationId actionCreate(Service service) String serviceName = service.getName(); ServiceApiUtil.validateNameFormat(serviceName, getConfig()); ServiceApiUtil.validateAndResolveService(service, fs, getConfig()); + String userName = ""; + if (UserGroupInformation.isSecurityEnabled()) { + try { + userName = UserGroupInformation.getCurrentUser().getUserName(); + } catch(KerberosAuthException e) { + userName = null; + } + } + ysc.deployApp(service, userName); verifyNoLiveAppInRM(serviceName, "create"); Path appDir = checkAppNotExistOnHdfs(service); @@ -429,6 +447,12 @@ public int actionDestroy(String serviceName) throws Exception { ServiceApiUtil.validateNameFormat(serviceName, getConfig()); verifyNoLiveAppInRM(serviceName, "destroy"); + String userName = ""; + if (UserGroupInformation.isSecurityEnabled()) { + userName = UserGroupInformation.getCurrentUser().getUserName(); + } + ysc.deleteApp(serviceName, userName); + Path appDir = fs.buildClusterDirPath(serviceName); FileSystem fileSystem = fs.getFileSystem(); // remove from the appId cache @@ -507,16 +531,7 @@ private int actionHelp(String actionName, CommonArgs args) private void verifyNoLiveAppInRM(String serviceName, String action) throws IOException, YarnException { - Set types = new HashSet<>(1); - types.add(YarnServiceConstants.APP_TYPE); - Set tags = null; - if (serviceName != null) { - tags = Collections.singleton(SliderUtils.createNameTag(serviceName)); - } - GetApplicationsRequest request = GetApplicationsRequest.newInstance(); - request.setApplicationTypes(types); - request.setApplicationTags(tags); - request.setApplicationStates(liveStates); + GetApplicationsRequest request = buildApplicationsRequest(serviceName); List reports = yarnClient.getApplications(request); if (!reports.isEmpty()) { String message = ""; @@ -531,6 +546,20 @@ private void verifyNoLiveAppInRM(String serviceName, String action) } } + protected GetApplicationsRequest buildApplicationsRequest(String serviceName) { + Set types = new HashSet<>(1); + types.add(YarnServiceConstants.APP_TYPE); + Set tags = null; + if (serviceName != null) { + tags = Collections.singleton(SliderUtils.createNameTag(serviceName)); + } + GetApplicationsRequest request = GetApplicationsRequest.newInstance(); + request.setApplicationTypes(types); + request.setApplicationTags(tags); + request.setApplicationStates(liveStates); + return request; + } + private ApplicationId submitApp(Service app) throws IOException, YarnException { String serviceName = app.getName(); @@ -824,6 +853,21 @@ public ServiceState convertState(FinalApplicationStatus status) { return ServiceState.ACCEPTED; } + /** + * Fetch service configuration from SOLR. + * + * @param serviceName - Service name + * @return Service specification + */ + public Service getSpec(String serviceName) throws IOException { + String userName = ""; + if (UserGroupInformation.isSecurityEnabled()) { + userName = UserGroupInformation.getCurrentUser().getUserName(); + } + Service appSpec = ysc.findAppEntry(serviceName, userName); + return appSpec; + } + public Service getStatus(String serviceName) throws IOException, YarnException { ServiceApiUtil.validateNameFormat(serviceName, getConfig()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/YarnSolrClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/YarnSolrClient.java new file mode 100644 index 0000000..77c72b2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/YarnSolrClient.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.client; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrInputDocument; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.conf.RestApiConstants; + +public class YarnSolrClient { + + private static final Log LOG = LogFactory.getLog(YarnSolrClient.class); + private static String SOLR_URL; + private static final String DEFAULT_SOLR_URL = + "http://localhost:8983/solr/yarn"; + private static boolean ENABLED; + + public YarnSolrClient() { + super(); + } + + static { + init(); + } + + private static void init() { + YarnConfiguration conf = new YarnConfiguration(); + SOLR_URL = conf.get(RestApiConstants.YARN_SOLR_URL_KEY, + DEFAULT_SOLR_URL); + ENABLED = conf.getBoolean(RestApiConstants.YARN_SOLR_STORAGE_ENABLED, + false); + } + + public SolrClient getSolrClient() { + return new HttpSolrClient.Builder(SOLR_URL).build(); + } + + public Service findAppEntry(String appName, String username) { + Service entry = new Service(); + if (!ENABLED) { + return entry; + } + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, + false); + + SolrClient solr = getSolrClient(); + SolrQuery query = new SolrQuery(); + query.setQuery("id:" + appName); + query.setFilterQueries("type_s:AppEntry"); + query.setRows(1); + + QueryResponse response; + try { + response = solr.query(query); + Iterator appList = response.getResults() + .listIterator(); + while (appList.hasNext()) { + SolrDocument d = appList.next(); + entry = mapper.readValue(d.get("yarnfile_s").toString(), + Service.class); + } + } catch (SolrServerException | IOException e) { + LOG.error("Error in finding deployed application: " + appName, e); + } + return entry; + } + + public Service deployApp(Service yarnApp, String username) throws + IOException { + if (!ENABLED) { + return yarnApp; + } + SolrClient solr = getSolrClient(); + Collection docs = + new HashSet(); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, + false); + + if (yarnApp!=null) { + // Register deployed application instance with AppList + SolrInputDocument request = new SolrInputDocument(); + request.addField("type_s", "AppEntry"); + request.addField("id", yarnApp.getName()); + request.addField("user_s", username); + request.addField("yarnfile_s", + mapper.writeValueAsString(yarnApp)); + docs.add(request); + } + + // Commit Solr changes. + try { + UpdateResponse detailsResponse = solr.add(docs); + if (detailsResponse.getStatus() != 0) { + throw new IOException("Unable to store spec file."); + } + solr.commit(); + } catch (SolrServerException e) { + LOG.error("Fail to store service configuration", e); + } + return yarnApp; + } + + public void deleteApp(String id, String username) { + if (!ENABLED) { + return; + } + SolrClient solr = getSolrClient(); + try { + solr.deleteById(id); + solr.commit(); + } catch (SolrServerException | IOException e) { + LOG.error("Error in removing deployed application: "+id, e); + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java index 6de2dc0..8d511f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java @@ -24,6 +24,8 @@ String VERSION = "/services/version"; String SERVICE_ROOT_PATH = "/services"; String SERVICE_PATH = "/services/{service_name}"; + String SERVICE_SPEC_PATH = "/services/{service_name}/spec"; + String SERVICE_STATUS_PATH = "/services/{service_name}/status"; String COMPONENT_PATH = "/services/{service_name}/components/{component_name}"; // Query param @@ -40,4 +42,7 @@ Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002; Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003; Integer ERROR_CODE_APP_NAME_INVALID = 404004; + + String YARN_SOLR_URL_KEY = "yarn.api-service.solr.url"; + String YARN_SOLR_STORAGE_ENABLED = "yarn.api-service.solr.storage.enabled"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java index 672c635..4898dea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java @@ -31,6 +30,7 @@ import org.apache.hadoop.yarn.service.api.records.Container; import org.apache.hadoop.yarn.service.api.records.ContainerState; import org.apache.hadoop.yarn.service.client.ServiceClient; +import org.apache.hadoop.yarn.service.client.YarnSolrClientTest; import org.apache.hadoop.yarn.service.exceptions.SliderException; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; import org.junit.After; @@ -78,9 +78,12 @@ public void tearDown() throws IOException { // End-to-end test to use ServiceClient to deploy a service. // 1. Create a service with 2 components, each of which has 2 containers - // 2. Flex up each component to 3 containers and check the component instance names - // 3. Flex down each component to 1 container and check the component instance names - // 4. Flex up each component to 2 containers and check the component instance names + // 2. Flex up each component to 3 containers and check the component + // instance names + // 3. Flex down each component to 1 container and check the component + // instance names + // 4. Flex up each component to 2 containers and check the component + // instance names // 5. Stop the service // 6. Destroy the service @Test (timeout = 200000) @@ -192,7 +195,8 @@ private void checkContainerLaunchDependencies(ServiceClient client, Map compCounts = new HashMap<>(); compCounts.put("compa", count); compCounts.put("compb", count); - // flex will update the persisted conf to reflect latest number of containers. + // flex will update the persisted conf to reflect latest number of + // containers. exampleApp.getComponent("compa").setNumberOfContainers(count); exampleApp.getComponent("compb").setNumberOfContainers(count); client.flexByRestService(exampleApp.getName(), compCounts); @@ -200,9 +204,11 @@ private void checkContainerLaunchDependencies(ServiceClient client, } // Check each component's comp instances name are in sequential order. - // E.g. If there are two instances compA-1 and compA-2 - // When flex up to 4 instances, it should be compA-1 , compA-2, compA-3, compA-4 - // When flex down to 3 instances, it should be compA-1 , compA-2, compA-3. + // E.g. If there are two instances compA-1 and compA-2. + // When flex up to 4 instances, it should be compA-1 , compA-2, + // compA-3, compA-4. + // When flex down to 3 instances, it should be compA-1 , + // compA-2, compA-3. private void checkCompInstancesInOrder(ServiceClient client, Service exampleApp) throws IOException, YarnException { Service service = client.getStatus(exampleApp.getName()); @@ -300,12 +306,17 @@ private void waitForAllCompToBeReady(ServiceClient client, private ServiceClient createClient() throws Exception { ServiceClient client = new ServiceClient() { - @Override protected Path addJarResource(String appName, + @Override + protected Path addJarResource(String appName, Map localResources) throws IOException, SliderException { // do nothing, the Unit test will use local jars return null; } + @Override + protected void createYarnSolrClient() { + ysc = new YarnSolrClientTest(); + } }; client.init(getConf()); client.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/EmbeddedSolrServerFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/EmbeddedSolrServerFactory.java new file mode 100644 index 0000000..59f0cf4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/EmbeddedSolrServerFactory.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.service.client; + +import org.apache.commons.io.FileUtils; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.core.NodeConfig; +import org.apache.solr.core.SolrResourceLoader; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class EmbeddedSolrServerFactory { + + /** + * Cleans the given solrHome directory and creates a new + * EmbeddedSolrServer. + * + * @param solrHome + * the Solr home directory to use + * @param configSetHome + * the directory containing config sets + * @param coreName + * the name of the core, must have a matching directory in + * configHome + * + * @return an EmbeddedSolrServer with a core created for the given + * coreName + * @throws IOException + */ + public static SolrClient create(final String solrHome, + final String configSetHome, final String coreName) + throws IOException, SolrServerException { + return create(solrHome, configSetHome, coreName, true); + } + + /** + * @param solrHome + * the Solr home directory to use + * @param configSetHome + * the directory containing config sets + * @param coreName + * the name of the core, must have a matching directory in + * configHome + * @param cleanSolrHome + * if true the directory for solrHome will be deleted and + * re-created if it already exists + * + * @return an EmbeddedSolrServer with a core created for the given + * coreName + * @throws IOException + */ + public static SolrClient create(final String solrHome, + final String configSetHome, final String coreName, + final boolean cleanSolrHome) throws IOException, + SolrServerException { + + final File solrHomeDir = new File(solrHome); + if (solrHomeDir.exists()) { + FileUtils.deleteDirectory(solrHomeDir); + solrHomeDir.mkdirs(); + } else { + solrHomeDir.mkdirs(); + } + + final SolrResourceLoader loader = new SolrResourceLoader( + solrHomeDir.toPath()); + final Path configSetPath = Paths + .get(configSetHome).toAbsolutePath(); + + final NodeConfig config = new NodeConfig.NodeConfigBuilder( + "embeddedSolrServerNode", loader) + .setConfigSetBaseDirectory(configSetPath + .toString()).build(); + + final EmbeddedSolrServer embeddedSolrServer = + new EmbeddedSolrServer(config, coreName); + + final CoreAdminRequest.Create createRequest = + new CoreAdminRequest.Create(); + createRequest.setCoreName(coreName); + createRequest.setConfigSet(coreName); + embeddedSolrServer.request(createRequest); + + return embeddedSolrServer; + } + +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/YarnSolrClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/YarnSolrClientTest.java new file mode 100644 index 0000000..7c2d406 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/YarnSolrClientTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.client; + +import org.apache.solr.client.solrj.SolrClient; + +public class YarnSolrClientTest extends YarnSolrClient { + + static final String CONFIGSET_DIR = "src/test/resources/configsets"; + static SolrClient solrClient; + + static String targetLocation = EmbeddedSolrServerFactory.class + .getProtectionDomain().getCodeSource().getLocation().getFile() + + "/.."; + + static String solrHome = targetLocation + "/solr"; + + public YarnSolrClientTest() { + super(); + } + + @Override + public SolrClient getSolrClient() { + try { + solrClient = EmbeddedSolrServerFactory.create(solrHome, + CONFIGSET_DIR, "exampleCollection"); + } catch (Exception e) { + } + return solrClient; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt new file mode 100644 index 0000000..2c164c0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +# Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +such +that +the +their +then +there +these +they +this +to +was +will +with diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/params.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/params.json new file mode 100644 index 0000000..06114ef --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/params.json @@ -0,0 +1,20 @@ +{"params":{ + "query":{ + "defType":"edismax", + "q.alt":"*:*", + "rows":"10", + "fl":"*,score", + "":{"v":0} + }, + "facets":{ + "facet":"on", + "facet.mincount": "1", + "":{"v":0} + }, + "velocity":{ + "wt": "velocity", + "v.template":"browse", + "v.layout": "layout", + "":{"v":0} + } +}} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/protwords.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/protwords.txt new file mode 100644 index 0000000..1dfc0ab --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/schema.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/schema.xml new file mode 100644 index 0000000..90dc4b8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/schema.xml @@ -0,0 +1,116 @@ + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml new file mode 100644 index 0000000..392feec --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml @@ -0,0 +1,36 @@ + + + + 6.2.1 + + ${solr.data.dir:} + + + + + single + + + + + + + + + + \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/stopwords.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/stopwords.txt new file mode 100644 index 0000000..ae1e83e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/stopwords.txt @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/synonyms.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/synonyms.txt new file mode 100644 index 0000000..7f72128 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/configsets/exampleCollection/conf/synonyms.txt @@ -0,0 +1,29 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaafoo => aaabar +bbbfoo => bbbfoo bbbbar +cccfoo => cccbar cccbaz +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 892f9f5..6f9e87d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3375,4 +3375,20 @@ + + yarn.api-service.solr.storage.enabled + false + + Flag to enable YARN Services for storing service configuration + on solr. + + + + + yarn.api-service.solr.url + http://localhost:8983/solr/yarn + + URL to Solr server for storing YARN Service config. + +