commit 8f4e06ef66ca3cc2d612b5c991601398ad32931a Author: Eric Yang Date: Wed Sep 13 16:48:19 2017 -0700 YARN-7129. Implemented REST for registering YARN application in application catalog. diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 31ac611f65d..29194f168d4 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -152,12 +152,19 @@ ENV MAVEN_OPTS -Xms256m -Xmx512m ### # Install node js tools for web UI frameowkr ### -RUN apt-get -y install nodejs && \ - ln -s /usr/bin/nodejs /usr/bin/node && \ +RUN mkdir -p /opt/nodejs && \ + curl -L -s -S \ + https://nodejs.org/download/release/v6.11.2/node-v6.11.2-linux-x64.tar.gz \ + -o /opt/node-v6.11.2-linux-x64.tar.gz && \ + tar xzf /opt/node-v6.11.2-linux-x64.tar.gz --strip-components 1 -C /opt/nodejs && \ + ln -s /opt/nodejs/bin/node /usr/bin/nodejs && \ + ln -s /opt/nodejs/bin/node /usr/bin/node && \ apt-get -y install npm && \ npm install npm@latest -g && \ npm install -g bower && \ - npm install -g ember-cli + npm install -g ember-cli && \ + npm install -g phantomjs && \ + npm install -g apidoc ### # Everything past this point is either not needed for testing or breaks Yetus. diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index c3366a5cc1b..d4d799c2f44 100755 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -431,6 +431,12 @@ org.apache.hadoop + hadoop-yarn-services-api + ${project.version} + + + + org.apache.hadoop hadoop-mapreduce-client-jobclient ${project.version} test-jar @@ -768,7 +774,7 @@ commons-io commons-io - 2.4 + 2.5 @@ -976,7 +982,7 @@ org.mockito mockito-all - 1.8.5 + 1.9.5 org.mock-server diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml index 74d9681bee4..5910d9ef6ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml @@ -23,9 +23,15 @@ hadoop-yarn-services-api Apache Hadoop YARN Services API + 3.1.0-SNAPSHOT jar Hadoop YARN REST APIs for services + + 1.5.6 + 6.2.0 + + @@ -42,6 +48,61 @@ + + + org.apache.rat + apache-rat-plugin + + + src/test/resources/configsets/exampleCollection/conf/params.json + + + + + org.apache.maven.plugins + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + + + api + X + + + apiGroup + X + + + apiName + X + + + apiParam + X + + + apiParamExample + X + + + apiSuccess + X + + + apiSuccessExample + X + + + apiError + X + + + apiErrorExample + X + + + + org.apache.maven.plugins maven-jar-plugin @@ -74,9 +135,32 @@ ${java.home} + + file:${project.build.testOutputDirectory}/log4j.properties + + + org.codehaus.mojo + exec-maven-plugin + 1.3.2 + + + apidocs + package + + exec + + + ${project.build.directory}/apidocs.log + apidoc + -i src/main/java -o ${project.build.directory}/site/apidocs + true + + + + @@ -116,5 +200,209 @@ javax.ws.rs jsr311-api + + + org.apache.hadoop + hadoop-common + test-jar + test + + + + junit + junit + 4.11 + test + + + org.mockito + mockito-all + 1.9.5 + test + + + org.powermock + powermock-module-junit4 + ${powermock.version} + + + org.javassist + javassist + + + test + + + org.powermock + powermock-api-mockito + ${powermock.version} + + + org.hamcrest + hamcrest-core + + + org.objenesis + objenesis + + + test + + + com.fasterxml.jackson.module + jackson-module-jaxb-annotations + 2.7.8 + + + + commons-io + commons-io + 2.5 + + + + org.glassfish.jersey.media + jersey-media-json-jackson + 2.12 + + + org.eclipse.jetty + jetty-xml + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-base + + + test + + + + com.sun.jersey + jersey-client + 1.19 + provided + + + + com.sun.jersey + jersey-server + 1.19 + provided + + + + asm + asm-commons + 3.3.1 + + + + org.apache.solr + solr-solrj + ${solr.version} + + + org.eclipse.jetty + jetty-xml + + + org.slf4j + jcl-over-slf4j + + + + + org.apache.solr + solr-core + ${solr.version} + + + org.eclipse.jetty + jetty-http + + + org.eclipse.jetty + jetty-io + + + org.eclipse.jetty + jetty-security + + + org.eclipse.jetty + jetty-xml + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + slf4j-api + + + test + + + + org.apache.solr + solr-test-framework + ${solr.version} + + + org.eclipse.jetty + jetty-http + + + org.eclipse.jetty + jetty-io + + + org.eclipse.jetty + jetty-security + + + org.eclipse.jetty + jetty-xml + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + slf4j-api + + + test + + + + com.fasterxml.jackson.core + jackson-core + 2.7.8 + provided + + + + com.fasterxml.jackson.core + jackson-annotations + 2.7.8 + provided + + + + + + disable-java8-doclint + + [1.8,) + + + -Xdoclint:none + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/Appcatalog.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/Appcatalog.java new file mode 100644 index 00000000000..05ec105211b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/Appcatalog.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.application; + +import javax.ws.rs.ApplicationPath; +import javax.ws.rs.core.Application; + +import org.apache.hadoop.yarn.appcatalog.controller.AppDetailsController; + +import java.util.Set; + +@ApplicationPath("service") +public class Appcatalog extends Application { + + @Override + public Set> getClasses() { + final Set> resources = new java.util.HashSet<>(); + // following code can be used to customize Jersey 2.0 JSON provider: + try { + final Class jsonProvider = Class.forName("org.glassfish.jersey.jackson.JacksonFeature"); + // Class jsonProvider = + // Class.forName("org.glassfish.jersey.moxy.json.MoxyJsonFeature"); + // Class jsonProvider = + // Class.forName("org.glassfish.jersey.jettison.JettisonFeature"); + resources.add(jsonProvider); + } catch (final ClassNotFoundException ex) { + ex.printStackTrace(); + } + addRestResourceClasses(resources); + return resources; + } + + /** + * Add your own resources here. + */ + private void addRestResourceClasses(final Set> resources) { + resources.add(AppDetailsController.class); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnClient.java new file mode 100644 index 00000000000..f9be9a5cb3a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnClient.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.application; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.appcatalog.model.AppEntry; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.client.ServiceClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.sun.jersey.api.client.UniformInterfaceException; + +public class YarnClient { + + private static final Logger LOG = + LoggerFactory.getLogger(YarnClient.class); + + private static Configuration YARN_CONFIG = new YarnConfiguration(); + private static ServiceClient SERVICE_CLIENT; + + public YarnClient() { + } + + static { + init(); + } + + private static void init() { + SERVICE_CLIENT = new ServiceClient(); + SERVICE_CLIENT.init(YARN_CONFIG); + SERVICE_CLIENT.start(); + } + + public static void createApp(Service app) { + try { + SERVICE_CLIENT.actionCreate(app); + } catch (IOException | YarnException e) { + LOG.error("Error in deploying application: ", e); + throw new RuntimeException("Failed to create application: " + app.getName() + " error: " + e.getMessage()); + } + } + + public static void deleteApp(String appName) { + boolean destroy = true; + try { + SERVICE_CLIENT.actionStop(appName, destroy); + if (destroy) { + SERVICE_CLIENT.actionDestroy(appName); + LOG.info("Successfully deleted service {}", appName); + } else { + LOG.info("Successfully stopped service {}", appName); + } + } catch (Exception e) { + LOG.error("Error in deleting application: ", e); + throw new RuntimeException("Failed to delete application: " + appName + " error: " + e.getMessage()); + } + } + + public static void restartApp(Service app) throws JsonProcessingException { + String appName = app.getName(); + try { + int result = SERVICE_CLIENT.actionStart(appName); + if (result != 0) { + throw new RuntimeException("Failed to restart application: " + appName); + } + } catch (IOException | YarnException | UniformInterfaceException e) { + LOG.error("Error in stopping application: ", e); + } + + } + + public static void stopApp(Service app) throws JsonProcessingException { + String appName = app.getName(); + try { + SERVICE_CLIENT.actionStop(appName, true); + } catch (IOException | YarnException | UniformInterfaceException e) { + LOG.error("Error in stopping application: ", e); + } + } + + public static void getStatus(AppEntry entry) { + try { + Service app = SERVICE_CLIENT.getStatus(entry.getName()); + entry.setYarnfile(app); + } catch (UniformInterfaceException | IOException | YarnException e) { + LOG.error("Error in fetching application status: ", e); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnSolrClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnSolrClient.java new file mode 100644 index 00000000000..e18ed3b22b2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnSolrClient.java @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.application; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.appcatalog.model.AppEntry; +import org.apache.hadoop.yarn.appcatalog.model.AppCatalogEntry; +import org.apache.hadoop.yarn.appcatalog.model.Application; +import org.apache.hadoop.yarn.appcatalog.utils.RandomWord; +import org.apache.hadoop.yarn.appcatalog.utils.WordLengthException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrInputDocument; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class YarnSolrClient { + + private static final Log LOG = LogFactory.getLog(YarnSolrClient.class); + private static String SOLR_URL; + + public YarnSolrClient() { + } + + static { + init(); + } + + public static void init() { + YarnConfiguration conf = new YarnConfiguration(); + SOLR_URL = "http://" + conf.get("yarn.apiserver.webapp.address", "localhost:8983/solr/appcatalog"); + } + + public SolrClient getSolrClient() { + return new HttpSolrClient.Builder(SOLR_URL).build(); + } + + public List getRecommendedApps() { + List apps = new ArrayList(); + SolrClient solr = getSolrClient(); + SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + query.setFilterQueries("type_s:AppCatalogEntry"); + query.setRows(40); + QueryResponse response; + try { + response = solr.query(query); + Iterator list = response.getResults().listIterator(); + while (list.hasNext()) { + SolrDocument d = list.next(); + AppCatalogEntry entry = new AppCatalogEntry(); + entry.setId(d.get("id").toString()); + entry.setOrg(d.get("org_s").toString()); + entry.setName(d.get("name_s").toString()); + entry.setDesc(d.get("desc_s").toString()); + if (d.get("icon_s")!=null) { + entry.setIcon(d.get("icon_s").toString()); + } + entry.setLike(Integer.parseInt(d.get("like_i").toString())); + entry.setDownload(Integer.parseInt(d.get("download_i").toString())); + apps.add(entry); + } + } catch (SolrServerException | IOException e) { + LOG.error("Error getting a list of recommended applications: ", e); + } + return apps; + } + + public List search(String keyword) { + List apps = new ArrayList(); + SolrClient solr = getSolrClient(); + SolrQuery query = new SolrQuery(); + if (keyword.length()==0) { + query.setQuery("*:*"); + query.setFilterQueries("type_s:AppCatalogEntry"); + } else { + query.setQuery(keyword); + query.setFilterQueries("type_s:AppCatalogEntry"); + } + query.setRows(40); + QueryResponse response; + try { + response = solr.query(query); + Iterator list = response.getResults().listIterator(); + while (list.hasNext()) { + SolrDocument d = list.next(); + AppCatalogEntry entry = new AppCatalogEntry(); + entry.setId(d.get("id").toString()); + entry.setOrg(d.get("org_s").toString()); + entry.setName(d.get("name_s").toString()); + entry.setDesc(d.get("desc_s").toString()); + entry.setLike(Integer.parseInt(d.get("like_i").toString())); + entry.setDownload(Integer.parseInt(d.get("download_i").toString())); + apps.add(entry); + } + } catch (SolrServerException | IOException e) { + LOG.error("Error in searching for applications: ", e); + } + return apps; + } + + public List listAppEntries() { + List list = new ArrayList(); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + SolrClient solr = getSolrClient(); + SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + query.setFilterQueries("type_s:AppEntry"); + query.setRows(40); + QueryResponse response; + try { + response = solr.query(query); + Iterator appList = response.getResults().listIterator(); + while (appList.hasNext()) { + SolrDocument d = appList.next(); + AppEntry entry = new AppEntry(); + entry.setId(d.get("id").toString()); + entry.setName(d.get("name_s").toString()); + entry.setApp(d.get("app_s").toString()); + entry.setYarnfile(mapper.readValue(d.get("yarnfile_s").toString(), org.apache.hadoop.yarn.service.api.records.Service.class)); + list.add(entry); + } + } catch (SolrServerException | IOException e) { + LOG.error("Error in listing deployed applications: ", e); + } + return list; + } + + public AppEntry findAppEntry(String id) { + AppEntry entry = new AppEntry(); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + SolrClient solr = getSolrClient(); + SolrQuery query = new SolrQuery(); + query.setQuery("id:" + id); + query.setFilterQueries("type_s:AppEntry"); + query.setRows(1); + + QueryResponse response; + try { + response = solr.query(query); + Iterator appList = response.getResults().listIterator(); + while (appList.hasNext()) { + SolrDocument d = appList.next(); + entry.setId(d.get("id").toString()); + entry.setApp(d.get("app_s").toString()); + entry.setName(d.get("name_s").toString()); + entry.setYarnfile(mapper.readValue(d.get("yarnfile_s").toString(), org.apache.hadoop.yarn.service.api.records.Service.class)); + } + } catch (SolrServerException | IOException e) { + LOG.error("Error in finding deployed application: " + id, e); + } + return entry; + } + + public org.apache.hadoop.yarn.service.api.records.Service deployApp(String id) throws SolrServerException, IOException { + long download = 0; + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + Collection docs = new HashSet(); + SolrClient solr = getSolrClient(); + // Find application information from AppCatalog + String name; + try { + Random r = new Random(); + int Low = 3; + int High = 10; + int seed = r.nextInt(High-Low) + Low; + int seed2 = r.nextInt(High-Low) + Low; + name = RandomWord.getNewWord(seed).toLowerCase() + "-" + RandomWord.getNewWord(seed2).toLowerCase(); + } catch (WordLengthException e) { + name = "c" + java.util.UUID.randomUUID().toString().substring(0, 11); + } + org.apache.hadoop.yarn.service.api.records.Service yarnApp = null; + SolrQuery query = new SolrQuery(); + query.setQuery("id:" + id); + query.setFilterQueries("type_s:AppCatalogEntry"); + query.setRows(1); + QueryResponse response = solr.query(query); + Iterator appList = response.getResults().listIterator(); + AppCatalogEntry entry = new AppCatalogEntry(); + while (appList.hasNext()) { + SolrDocument d = appList.next(); + entry.setOrg(d.get("org_s").toString()); + entry.setName(d.get("name_s").toString()); + entry.setDesc(d.get("desc_s").toString()); + yarnApp = mapper.readValue(d.get("yarnfile_s").toString(), org.apache.hadoop.yarn.service.api.records.Service.class); + entry.setLike(Integer.parseInt(d.get("like_i").toString())); + entry.setDownload(Integer.parseInt(d.get("download_i").toString())); + download = entry.getDownload() + 1; + + // Update download count + docs.add(incrementDownload(d, download)); + } + + if (yarnApp!=null) { + // Register deployed application instance with AppList + yarnApp.setName(name); + SolrInputDocument request = new SolrInputDocument(); + request.addField("type_s", "AppEntry"); + request.addField("id", name); + request.addField("name_s", name); + request.addField("app_s", entry.getOrg()+"/"+entry.getName()); + request.addField("yarnfile_s", mapper.writeValueAsString(yarnApp)); + docs.add(request); + } + + // Commit Solr changes. + UpdateResponse detailsResponse = solr.add(docs); + if (detailsResponse.getStatus() != 0) { + throw new IOException("Unable to register docker instance with application entry."); + } + solr.commit(); + return yarnApp; + } + + private SolrInputDocument incrementDownload(SolrDocument doc, long download) { + Collection names = doc.getFieldNames(); + SolrInputDocument s = new SolrInputDocument(); + for (String name : names) { + if(!name.equals("_version_")) { + s.addField(name, doc.getFieldValues(name)); + } + } + download++; + s.setField("download_i", download); + return s; + } + + public void deleteApp(String id) { + SolrClient solr = getSolrClient(); + try { + solr.deleteById(id); + solr.commit(); + } catch (SolrServerException | IOException e) { + LOG.error("Error in removing deployed application: "+id, e); + } + } + + public void register(Application app) throws IOException { + Collection docs = new HashSet(); + SolrClient solr = getSolrClient(); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + try { + SolrInputDocument buffer = new SolrInputDocument(); + buffer.setField("id", java.util.UUID.randomUUID().toString().substring(0, 11)); + buffer.setField("org_s", app.getOrganization()); + buffer.setField("name_s", app.getName()); + buffer.setField("desc_s", app.getDescription()); + if (app.getIcon() != null) { + buffer.setField("icon_s", app.getIcon()); + } + buffer.setField("type_s", "AppCatalogEntry"); + buffer.setField("like_i", 0); + buffer.setField("download_i", 0); + + // Keep only YARN data model for yarnfile field + String yarnFile = mapper.writeValueAsString(app); + LOG.info("app:"+yarnFile); + org.apache.hadoop.yarn.service.api.records.Service yarnApp = mapper.readValue(yarnFile, org.apache.hadoop.yarn.service.api.records.Service.class); + buffer.setField("yarnfile_s", mapper.writeValueAsString(yarnApp)); + + docs.add(buffer); + // Commit Solr changes. + UpdateResponse detailsResponse = solr.add(docs); + if (detailsResponse.getStatus() != 0) { + throw new IOException("Unable to register application in Application Catalog."); + } + solr.commit(); + } catch (SolrServerException | IOException e) { + throw new IOException("Unable to register application in Application Catalog. "+ e.getMessage()); + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppCatalogController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppCatalogController.java new file mode 100644 index 00000000000..508f99dbf25 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppCatalogController.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.controller; + +import java.io.IOException; +import java.util.List; + +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + +import org.apache.hadoop.yarn.appcatalog.application.YarnSolrClient; +import org.apache.hadoop.yarn.appcatalog.model.AppCatalogEntry; +import org.apache.hadoop.yarn.appcatalog.model.Application; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +@Path("/ws/v2/app_catalog") +public class AppCatalogController { + + public AppCatalogController() { + } + + /** + * Display the most frequently used applications on appcatalog home page. + * + * @apiGroup AppCatalogController + * @apiName get + * @api {get} /ws/v2/app_catalog/recommended Display recommended applications. + * @apiSuccess {Object} AppEntry Application configuration. + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * [ + * { + * "id":"96b7833a-e3", + * "org":"Hortonworks", + * "name":"LAMP", + * "desc":"Linux Apache MySQL PHP web application", + * "icon":"/css/img/hwx_logo.png", + * "like":0, + * "download":0, + * "app":null + * }, + * { + * ... + * } + * ] + * @return - List of YARN applications + */ + @GET + @Path("recommended") + @Produces(MediaType.APPLICATION_JSON) + public List get() { + YarnSolrClient sc = new YarnSolrClient(); + return sc.getRecommendedApps(); + } + + /** + * Search for yarn applications from solr. + * + * @apiGroup AppCatalogController + * @apiName search + * @api {get} /ws/v2/app_catalog/search Find application from appcatalog. + * @apiParam {String} q Keyword to search. + * @apiSuccess {Object} AppCatalogEntry List of matched applications. + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * [ + * { + * "id":"96b7833a-e3", + * "org":"Hortonworks", + * "name":"LAMP", + * "desc":"Linux Apache MySQL PHP web application", + * "icon":"/css/img/hwx_logo.png", + * "like":0, + * "download":0, + * "app":null + * }, + * { + * ... + * } + * ] + * @param keyword - search for keyword + * @return - List of YARN applications matching keyword search. + */ + @GET + @Path("search") + @Produces(MediaType.APPLICATION_JSON) + public List search(@QueryParam("q") String keyword) { + YarnSolrClient sc = new YarnSolrClient(); + return sc.search(keyword); + } + + /** + * Register an application + * + * @apiGroup AppCatalogController + * @apiName register + * @api {post} /ws/v2/app_catalog/register Register an application in appcatalog. + * @apiParam {Object} app Application definition. + * @apiParamExample {json} Request-Example: + * { + * "name": "Jenkins", + * "organization": "Jenkins-ci.org", + * "description": "The leading open source automation server", + * "icon": "/css/img/jenkins.png", + * "lifetime": "3600", + * "components": [ + * { + * "name": "jenkins", + * "number_of_containers": 1, + * "artifact": { + * "id": "eyang-1.openstacklocal:5000/jenkins:latest", + * "type": "DOCKER" + * }, + * "launch_command": "", + * "resource": { + * "cpus": 1, + * "memory": "2048" + * }, + * "configuration": { + * "env": { + * }, + * "files": [ + * ] + * } + * } + * ], + * "quicklinks": { + * "Jenkins UI": "http://jenkins.${SERVICE_NAME}.${USER}.${DOMAIN}:8080/" + * } + * } + * @apiSuccess {String} Response Application register result. + * @apiError BadRequest Error in process application registration. + * @param app - Yarnfile in JSON form + * @return Web response + */ + @POST + @Path("register") + @Produces(MediaType.APPLICATION_JSON) + public Response register(Application app) { + try { + if (app.getName()==null) { + throw new IOException("Application name can not be empty."); + } + if (app.getOrganization()==null) { + throw new IOException("Application organization can not be empty."); + } + if (app.getDescription()==null) { + throw new IOException("Application description can not be empty."); + } + YarnSolrClient sc = new YarnSolrClient(); + sc.register(app); + } catch (IOException e) { + return Response.status(Status.BAD_REQUEST).entity(e.getMessage()).build(); + } + return Response.status(Status.ACCEPTED).build(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppDetailsController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppDetailsController.java new file mode 100644 index 00000000000..e9a23e24c19 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppDetailsController.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.controller; + +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.appcatalog.application.YarnClient; +import org.apache.hadoop.yarn.appcatalog.application.YarnSolrClient; +import org.apache.hadoop.yarn.appcatalog.model.AppEntry; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ServiceState; + +import com.fasterxml.jackson.core.JsonProcessingException; + +@Path("/ws/v2/app_details") +public class AppDetailsController { + + public AppDetailsController() { + } + + /** + * List detail information about the deployed application. + * + * @apiGroup AppDetailController + * @apiName getDetails + * @api {get} /ws/v2/app_details/config/{id} Check config of application instance. + * @apiParam {String} id Application ID to fetch configuration. + * @apiSuccess {Object} AppEntry Application configuration. + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "id":"howita_man", + * "name":"howita_man", + * "app":"Jenkins-ci.org/Jenkins", + * "yarnfile":{ + * "name":"howita_man", + * "lifetime":3600, + * "containers":[ + * ], + * "components":[ + * { + * "name":"jenkins", + * "dependencies":[ + * ], + * "artifact":{ + * "id":"eyang-1.openstacklocal:5000/jenkins:latest", + * "type":"DOCKER" + * }, + * "launch_command":"", + * "resource":{ + * "uri":null, + * "profile":null, + * "cpus":1, + * "memory":"2048" + * }, + * "number_of_containers":1, + * "run_privileged_container":false, + * "configuration":{ + * "properties":{ + * }, + * "env":{ + * }, + * "files":[ + * ] + * }, + * "quicklinks":[ + * ], + * "containers":[ + * ] + * } + * ], + * "configuration":{ + * "properties":{ + * }, + * "env":{ + * }, + * "files":[ + * ] + * }, + * "quicklinks":{ + * "Jenkins UI":"http://jenkins.howita_man.yarn.${DOMAIN}:8080/" + * } + * } + * } + * @param id - Application ID + * @return application entry- + */ + @Path("config/{id}") + @GET + @Produces(MediaType.APPLICATION_JSON) + public AppEntry getDetails(@PathParam("id") String id) { + YarnSolrClient sc = new YarnSolrClient(); + return sc.findAppEntry(id); + } + + /** + * Check application status + * + * @apiGroup AppDetailController + * @apiName getStatus + * @api {get} /ws/v2/app_details/status/{id} Check status of application instance. + * @apiParam {String} id Application ID to check. + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "id":"howita_man", + * "name":"howita_man", + * "app":"Jenkins-ci.org/Jenkins", + * "yarnfile":{ + * "name":"howita_man", + * "lifetime":3099, + * "containers":[ + * ], + * "id":"application_1503694240849_0043", + * "components":[ + * { + * "name":"jenkins", + * "dependencies":[ + * ], + * "artifact":{ + * "id":"eyang-1.openstacklocal:5000/jenkins:latest", + * "type":"DOCKER" + * }, + * "launch_command":"", + * "resource":{ + * "uri":null, + * "profile":null, + * "cpus":1, + * "memory":"2048" + * }, + * "number_of_containers":1, + * "run_privileged_container":false, + * "configuration":{ + * "properties":{ + * }, + * "env":{ + * }, + * "files":[ + * ] + * }, + * "quicklinks":[ + * ], + * "containers":[ + * { + * "id":"container_1503694240849_0043_01_000002", + * "launch_time":1504630535403, + * "bare_host":"eyang-4.openstacklocal", + * "state":"READY", + * "component_name":"jenkins-0" + * } + * ] + * } + * ], + * "configuration":{ + * "properties":{ + * }, + * "env":{ + * }, + * "files":[ + * ] + * }, + * "quicklinks":{ + * "Jenkins UI":"http://jenkins.howita_man.yarn.${DOMAIN}:8080/" + * } + * } + * } + * @apiSuccess {Object} text Give status + * @param id - Application ID + * @return application entry + */ + @Path("status/{id}") + @GET + @Produces(MediaType.APPLICATION_JSON) + public AppEntry getStatus(@PathParam("id") String id) { + YarnSolrClient sc = new YarnSolrClient(); + AppEntry appEntry = sc.findAppEntry(id); + YarnClient.getStatus(appEntry); + return appEntry; + } + + /** + * Stop an application + * + * @apiGroup AppDetailController + * @apiName stopApp + * @api {post} /ws/v2/app_details/stop/{id} Stop one instance of application. + * @apiParam {String} id Application ID to stop. + * @apiSuccess {String} text Give deployment status + * @apiError BadRequest Requested application does not stop. + * @param id - Application ID + * @return Web response code + */ + @Path("stop/{id}") + @POST + @Produces(MediaType.APPLICATION_JSON) + public Response stopApp(@PathParam("id") String id) { + YarnSolrClient sc = new YarnSolrClient(); + AppEntry app = sc.findAppEntry(id); + Service yarnApp = app.getYarnfile(); + yarnApp.setState(ServiceState.STOPPED); + try { + YarnClient.stopApp(yarnApp); + } catch (JsonProcessingException e) { + return Response.status(Status.BAD_REQUEST).build(); + } + return Response.ok().build(); + } + + /** + * Restart an application + * + * @apiGroup AppDetailController + * @apiName restartApp + * @api {post} /ws/v2/app_details/restart/{id} Restart one instance of application. + * @apiParam {String} id Application ID to restart. + * @apiSuccess {String} text Give deployment status + * @apiError BadRequest Requested application does not restart. + * @param id - Application ID + * @return Web response code + */ + @Path("restart/{id}") + @POST + @Produces(MediaType.APPLICATION_JSON) + public Response restartApp(@PathParam("id") String id) { + YarnSolrClient sc = new YarnSolrClient(); + AppEntry app = sc.findAppEntry(id); + Service yarnApp = app.getYarnfile(); + yarnApp.setState(ServiceState.STARTED); + try { + YarnClient.restartApp(yarnApp); + } catch (JsonProcessingException e) { + return Response.status(Status.BAD_REQUEST).build(); + } + return Response.ok().build(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppListController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppListController.java new file mode 100644 index 00000000000..c2368dccfbf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppListController.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.controller; + +import java.io.IOException; +import java.util.List; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.appcatalog.application.YarnClient; +import org.apache.hadoop.yarn.appcatalog.application.YarnSolrClient; +import org.apache.hadoop.yarn.appcatalog.model.AppEntry; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.solr.client.solrj.SolrServerException; + +/** + * Application deployment module + * + */ +@Path("/ws/v2/app_list") +@Produces({ MediaType.APPLICATION_JSON }) +public class AppListController { + + public AppListController() { + } + + /** + * Get Application List + * + * @apiGroup AppListController + * @apiName get + * @api {get} /ws/v2/app_list Get list of deployed applications. + * @apiSuccess {Object[]} List<AppEntry> List of deployed Applications. + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * [ + * { + * "id":"howita-man", + * "name":"howita-man", + * "app":"Jenkins-ci.org/Jenkins", + * "yarnfile":{ + * "name":"howita_man", + * "lifetime":3600, + * "containers":[ + * ], + * "components":[ + * { + * "name":"jenkins", + * "dependencies":[ + * ], + * "artifact":{ + * "id":"eyang-1.openstacklocal:5000/jenkins:latest", + * "type":"DOCKER" + * }, + * "launch_command":"", + * "resource":{ + * "uri":null, + * "profile":null, + * "cpus":1, + * "memory":"2048" + * }, + * "number_of_containers":1, + * "run_privileged_container":false, + * "configuration":{ + * "properties":{ + * }, + * "env":{ + * }, + * "files":[ + * ] + * }, + * "quicklinks":[ + * ], + * "containers":[ + * ] + * } + * ], + * "configuration":{ + * "properties":{ + * }, + * "env":{ + * }, + * "files":[ + * ] + * }, + * "quicklinks":{ + * "Jenkins UI":"http://jenkins.${SERVICE_NAME}.${USER}.${DOMAIN}:8080/" + * } + * } + * }, + * { + * ... + * } + * ] + * @return - Active application deployed by current user. + */ + @GET + @Produces(MediaType.APPLICATION_JSON) + public List getList() { + YarnSolrClient sc = new YarnSolrClient(); + return sc.listAppEntries(); + } + + /** + * Delete an application + * + * @apiGroup AppListController + * @apiName delete + * @api {delete} /ws/v2/app_list Delete one instance of application. + * @apiParam {String} id Application name to delete. + * @apiSuccess {String} text Delete request accepted + * @param id - application ID + * @param name - application name + * @return Web response + */ + @DELETE + @Path("{id}/{name}") + @Produces(MediaType.APPLICATION_JSON) + public Response delete(@PathParam("id") String id, @PathParam("name") String name) { + YarnSolrClient sc = new YarnSolrClient(); + sc.deleteApp(id); + YarnClient.deleteApp(name); + return Response.status(Status.ACCEPTED).build(); + } + + /** + * Deploy an application + * + * @apiGroup AppListController + * @apiName deploy + * @api {post} /ws/v2/app_list/{id} Deploy one instance of application. + * @apiParam {String} id Application ID to deploy. + * @apiSuccess {String} text Give deployment status + * @apiError BadRequest Unable to deploy requested application. + * @param id - application ID + * @return Web response + */ + @POST + @Path("{id}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public Response deploy(@PathParam("id") String id) { + YarnSolrClient sc = new YarnSolrClient(); + Service app; + try { + app = sc.deployApp(id); + } catch (SolrServerException | IOException e) { + return Response.status(Status.BAD_REQUEST).entity(e.toString()).build(); + } + YarnClient.createApp(app); + String output = "{\"status\":\"Application deployed.\",\"id\":\"" + app.getName() + "\"}"; + return Response.status(Status.ACCEPTED).entity(output).build(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppCatalogEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppCatalogEntry.java new file mode 100644 index 00000000000..b01155f3b4c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppCatalogEntry.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.PUBLIC_MEMBER) +public class AppCatalogEntry { + private String id; + private String org; + private String name; + private String desc; + private String icon = "css/img/feather.png"; + private long like; + private long download; + private Application app; + + public String getOrg() { + return org; + } + + public void setOrg(String org) { + this.org = org; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public long getLike() { + return like; + } + + public void setLike(long like) { + this.like = like; + } + + public long getDownload() { + return download; + } + + public void setDownload(long download) { + this.download = download; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Application getApp() { + return app; + } + + public void setApp(Application app) { + this.app = app; + } + + public String getIcon() { + return icon; + } + + public void setIcon(String icon) { + this.icon = icon; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java new file mode 100644 index 00000000000..e51453812c4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.model; + +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.PUBLIC_MEMBER) +public class AppDetails { + private String image; + private String version; + private List ports; + private List volumes; + private List env; + + public String getImage() { + return image; + } + + public void setImage(String image) { + this.image = image; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public List getPorts() { + return ports; + } + + public void setPorts(List ports2) { + this.ports = ports2; + } + + public List getVolumes() { + return volumes; + } + + public void setVolumes(List volumes) { + this.volumes = volumes; + } + + public List getEnv() { + return env; + } + + public void setEnv(List env) { + this.env = env; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppEntry.java new file mode 100644 index 00000000000..d25d3139588 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppEntry.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlType; + +@XmlType(namespace="http://hadoop.apache.org", name="AppEntry") +@XmlRootElement +@XmlAccessorType(XmlAccessType.PUBLIC_MEMBER) +public class AppEntry { + private String id; + private String name; + private String app; + private org.apache.hadoop.yarn.service.api.records.Service yarnfile; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getApp() { + return app; + } + + public void setApp(String app) { + this.app = app; + } + + public org.apache.hadoop.yarn.service.api.records.Service getYarnfile() { + return yarnfile; + } + + public void setYarnfile(org.apache.hadoop.yarn.service.api.records.Service yarnfile) { + this.yarnfile = yarnfile; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java new file mode 100644 index 00000000000..17d06cc69cb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.model; + +import javax.xml.bind.annotation.XmlRootElement; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@XmlRootElement +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonPropertyOrder({ "organization", "name", "description", "icon" }) +public class Application extends org.apache.hadoop.yarn.service.api.records.Service { + private static final long serialVersionUID = -1776203219305414248L; + + private String organization; + private String description; + private String icon; + + @JsonProperty("organization") + public String getOrganization() { + return organization; + } + public void setOrganization(String organization) { + this.organization = organization; + } + + @JsonProperty("description") + public String getDescription() { + return description; + } + public void setDescription(String description) { + this.description = description; + } + + @JsonProperty("icon") + public String getIcon() { + return icon; + } + public void setIcon(String icon) { + this.icon = icon; + } + + @Override + public boolean equals(Object o) { + return super.equals(o); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/utils/RandomWord.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/utils/RandomWord.java new file mode 100644 index 00000000000..adc95c2e430 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/utils/RandomWord.java @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.appcatalog.utils; + +import java.util.Arrays; +import java.util.Random; + +public class RandomWord { + + public static String getNewWord(int wordLength) throws WordLengthException { + if (wordLength < 3 || wordLength > 15) { + throw new WordLengthException("Word length error, words must be between 3 and 15 characters long."); + } + String theNewWord = "e"; + while (theNewWord.equals("e")) { + try { + theNewWord = generateRandomWord(wordLength); + } catch (Exception e) { + } + } + return theNewWord; + } + + private static String generateRandomWord(int wordLength) { + String randomWord; + // ----------------------------------------------------------------------------------------- + // Bigram source and general concept based on + // https://github.com/scrollback/scrollback & described in + // https://www.hackerearth.com/notes/random-pronouncable-text-generator/ + String[] startBiGram = { "TH", "OF", "AN", "IN", "TO", "CO", "BE", "HE", "RE", "HA", "WA", "FO", "WH", "MA", "WI", + "ON", "HI", "PR", "ST", "NO", "IS", "IT", "SE", "WE", "AS", "CA", "DE", "SO", "MO", "SH", "DI", "AL", "AR", + "LI", "WO", "FR", "PA", "ME", "AT", "SU", "BU", "SA", "FI", "NE", "CH", "PO", "HO", "DO", "OR", "UN", "LO", + "EX", "BY", "FA", "LA", "LE", "PE", "MI", "SI", "YO", "TR", "BA", "GO", "BO", "GR", "TE", "EN", "OU", "RA", + "AC", "FE", "PL", "CL", "SP", "BR", "EV", "TA", "DA", "AB", "TI", "RO", "MU", "EA", "NA", "SC", "AD", "GE", + "YE", "AF", "AG", "UP", "AP", "DR", "US", "PU", "CE", "IF", "RI", "VI", "IM", "AM", "KN", "OP", "CR", "OT", + "JU", "QU", "TW", "GA", "VA", "VE", "PI", "GI", "BI", "FL", "BL", "EL", "JO", "FU", "HU", "CU", "RU", "OV", + "MY", "OB", "KE", "EF", "PH", "CI", "KI", "NI", "SL", "EM", "SM", "VO", "MR", "WR", "ES", "DU", "TU", "AU", + "NU", "GU", "OW", "SY", "JA", "OC", "EC", "ED", "ID", "JE", "AI", "EI", "SK", "OL", "GL", "EQ", "LU", "AV", + "SW", "AW", "EY", "TY" }; + String[] lookupBiGram = { "TH", "AN", "IN", "IO", "EN", "TI", "FO", "HE", "HA", "HI", "TE", "AT", "ER", "AL", "WA", + "VE", "CO", "RE", "IT", "WI", "ME", "NC", "ON", "PR", "AR", "ES", "EV", "ST", "EA", "IV", "EC", "NO", "OU", + "PE", "IL", "IS", "MA", "AV", "OM", "IC", "GH", "DE", "AI", "CT", "IG", "ID", " OR", "OV", "UL", "YO", "BU", + "RA", "FR", "RO", "WH", "OT", "BL", "NT", "UN", "TR", "HO", "AC", "TU", "WE", "CA", "SH", "UR", "IE", "PA", + "TO", "EE", "LI", "RI", "UG", "AM", "ND", "US", "LL", "AS", "TA", "LE", "MO", "WO", "MI", "AB", "EL", "IA", + "NA", "SS", "AG", "TT", "NE", "PL", " LA", "OS", "CE", "DI", "BE", "AP", "SI", "NI", "OW", "SO", "AK", "CH", + "EM", "IM", "SE", "NS", "PO", "EI", "EX", "KI", "UC", "AD", "GR", "IR", "NG", "OP", "SP", "OL", "DA", "NL", + "TL", "LO", "BO", "RS", "FE", "FI", "SU", "GE", "MP", "UA", "OO", "RT", "SA", "CR", "FF", "IK", "MB", " KE", + "FA", "CI", "EQ", "AF", "ET", "AY", "MU", "UE", "HR", "TW", "GI", "OI", "VI", "CU", "FU", "ED", "QU", "UT", + "RC", "OF", "CL", "FT", "IZ", "PP", "RG", "DU", "RM", "YE", "RL", "DO", "AU", "EP", "BA", "JU", "RD", "RU", + "OG", "BR", "EF", "KN", "LS", "GA", "PI", "YI", "BI", "IB", "UB", "VA", "OC", "IF", "RN", "RR", "SC", "TC", + "CK", "DG", "DR", "MM", "NN", "OD", "RV", "UD", "XP", "JE", "UM", "EG", "DL", "PH", "SL", "GO", "CC", "LU", + "OA", "PU", "UI", "YS", "ZA", "HU", "MR", "OE", "SY", "EO", "TY", "UP", "FL", "LM", "NF", "RP", "OH", "NU", + "XA", "OB", "VO", "DM", "GN", " LD", "PT", "SK", "WR", "JO", "LT", "YT", "UF", "BJ", "DD", "EY", "GG", "GL", + "GU", "HT", "LV", "MS", "NM", "NV", "OK", "PM", "RK", "SW", "TM", "XC", "ZE", "AW", "SM" }; + String[][][] nextCharLookup = { { { "E", "A", "I", "O", "R" }, { "E", "O" } }, + { { "D", "T", "Y", "C", "S", "G", "N", "I", "O", "E", "A", "K" }, + { "D", "T", "Y", "S", "G", "O", "E", "A", "K" } }, + { { "G", "T", "E", "D", "S", "C", "A", "I", "K", "V", "U", "N", "F" }, { "G", "T", "E", "D", "S", "A", "K" } }, + { { "N", "U", "R" }, { "N", "U", "R" } }, + { { "T", "C", "D", "S", "E", "I", "G", "O", "N", "A" }, { "T", "D", "S", "E", "G", "O", "A" } }, + { { "O", "N", "C", "V", "M", "L", "E", "T", "S", "A", "R", "F" }, + { "N", "C", "M", "L", "E", "T", "S", "A", "R", "F" } }, + { { "R", "U", "O", "L" }, { "R", "U", "O", "L" } }, + { { "R", "N", "Y", "S", "M", "I", "A", "L", "D", "T" }, { "R", "N", "Y", "S", "M", "A", "L", "D", "T" } }, + { { "T", "D", "V", "N", "S", "R", "P", "L" }, { "T", "D", "N", "S", "R", "L" } }, + { { "S", "N", "C", "M", "L", "P", "G", "T", "R", "E" }, { "S", "N", "C", "M", "L", "P", "G", "T", "R", "E" } }, + { { "R", "D", "N", "S", "M", "L", "E", "C", "A" }, { "R", "D", "N", "S", "M", "L", "E", "A" } }, + { { "I", "E", "T", "H", "U", "O", "C" }, { "E", "H", "O" } }, + { { "E", "S", "I", "A", "N", "Y", "T", "V", "M", "R", "O", "L", "G", "F", "C" }, + { "E", "S", "A", "N", "Y", "T", "M" } }, + { { "L", "S", "I", "T", "E", "U", "O", "M", "K", "F", "A" }, { "L", "S", "T", "E", "F" } }, + { { "S", "Y", "R", "T", "N", "L" }, { "S", "Y", "R", "T", "N", "L" } }, + { { "R", "N", "L", "S", "D" }, { "R", "N", "L", "S", "D" } }, + { { "N", "M", "U", "R", "L", "V", "S", "O" }, { "N", "M", "U", "R", "L", "O" } }, + { { "S", "A", "D", "N", "E", "C", "L", "T", "P", "M", "V", "G", "F", "Q" }, + { "S", "A", "D", "N", "E", "L", "T", "P", "M" } }, + { { "H", "I", "Y", "E", "S", "T", "A", "U" }, { "H", "Y", "E", "S", "A" } }, + { { "T", "L", "N", "S" }, { "T", "L", "N", "S" } }, + { { "N", "R", "D", "T", "S", "M", "A" }, { "N", "R", "D", "T", "S", "M", "A" } }, + { { "E", "I", "H", "T", "R", "O", "L" }, { "E", "H", "T" } }, + { { "S", "E", "T", "G", "A", "D", "L", "C", "V", "O", "I", "F" }, { "S", "E", "T", "G", "A", "D", "O" } }, + { { "O", "E", "I", "A" }, { "E", "A" } }, + { { "E", "T", "D", "Y", "S", "I", "R", "L", "M", "K", "G", "A", "O", "N", "C" }, + { "E", "T", "D", "Y", "S", "M", "K", "A", "N" } }, + { { "S", "T", "E", "I", "P", "U", "C" }, { "S", "T", "E" } }, { { "E", "I" }, { "E" } }, + { { "A", "R", "I", "E", "O", "U", "S" }, { "A", "E", "O", "S" } }, + { { "R", "S", "T", "D", "L", "C", "N", "V", "M", "K" }, { "R", "S", "T", "D", "L", "N", "M" } }, + { { "E", "I", "A" }, { "E" } }, { { "T", "O", "I", "E", "A", "U", "R", "H" }, { "T", "E", "H" } }, + { { "T", "W", "R", "U", "N", "M" }, { "T", "W", "R", "U", "N", "M" } }, + { { "T", "L", "R", "N", "S", "G", "P", "B" }, { "T", "L", "R", "N", "S", "P" } }, + { { "R", "N", "C", "A", "D", "T", "O" }, { "R", "N", "A", "D", "T" } }, + { { "L", "E", "I", "Y", "D", "A" }, { "L", "E", "Y", "D" } }, + { { "T", "H", "S", "I", "E", "C", "M" }, { "T", "H", "S", "E", "M" } }, + { { "N", "T", "L", "K", "D", "S", "I", "G" }, { "N", "T", "L", "D", "S" } }, { { "E", "I", "A" }, { "E" } }, + { { "E", "P", "M", "I", "A" }, { "E" } }, + { { "A", "H", "E", "I", "T", "K", "U", "S" }, { "H", "E", "T", "K", "S" } }, { { "T" }, { "T" } }, + { { "R", "N", "S", "D", "A", "V", "P", "T", "M", "L", "F" }, { "R", "N", "S", "D", "A", "P", "T", "M", "L" } }, + { { "N", "D", "R", "L", "T" }, { "N", "D", "R", "L", "T" } }, + { { "I", "E", "U", "S", "O" }, { "E", "S", "O" } }, { { "H", "N", "I" }, { "H", "N" } }, { { "E" }, { "E" } }, + { { "E", "T", "M", "D", "S", "K", "I", "Y", "L", "G", "A", "R", "N", "C" }, + { "E", "T", "M", "D", "S", "K", "Y", "A", "N" } }, + { { "E", "I" }, { "E" } }, { { "D", "T", "A", "L" }, { "D", "T", "L" } }, { { "U" }, { "U" } }, + { { "T", "S", "R", "I" }, { "T", "S", "R" } }, + { { "T", "N", "L", "C", "I", "M", "D", "S", "R", "P", "G", "B" }, { "T", "N", "L", "M", "D", "S", "R" } }, + { { "O", "E", "A" }, { "E", "A" } }, + { { "M", "U", "V", "P", "N", "W", "S", "O", "L", "D", "C", "B", "A", "T", "G" }, + { "M", "U", "P", "N", "W", "O", "L", "D", "T" } }, + { { "I", "E", "O", "A" }, { "E", "O" } }, { { "H", "E", "T", "I" }, { "H", "E" } }, + { { "E", "I", "Y", "O", "A" }, { "E", "Y" } }, + { { "E", "I", "S", "R", "O", "A", "L", "Y", "U", "H" }, { "E", "S", "O", "A", "Y", "H" } }, + { { "D", "T", "I", "C", "G" }, { "D", "T", "G" } }, { { "A", "I", "O", "E", "U", "Y" }, { "A", "E", "Y" } }, + { { "U", "W", "S", "R", "L", "O", "M", "T", "P", "N", "D" }, + { "U", "W", "R", "L", "O", "M", "T", "P", "N", "D" } }, + { { "T", "E", "K", "H", "C", "R", "I" }, { "T", "E", "K", "H" } }, { { "R", "D", "A", "T" }, { "R", "T" } }, + { { "R", "L", "E", "V", "S", "N", "A" }, { "R", "L", "E", "S", "N", "A" } }, + { { "L", "N", "T", "R", "U", "S", "M", "P" }, { "L", "N", "T", "R", "S", "M" } }, + { { "E", "O", "I", "A" }, { "E", "O" } }, + { { "E", "N", "T", "S", "I", "A", "Y", "R", "P", "C" }, { "E", "N", "T", "S", "A", "Y" } }, + { { "S", "N", "D", "T", "W", "V", "R", "L", "F" }, { "S", "N", "D", "T", "W", "R", "L" } }, + { { "R", "T", "S", "N", "L", "I", "C" }, { "R", "T", "S", "N", "L" } }, + { { "R", "O", "N", "W", "P", "M", "L" }, { "R", "O", "N", "W", "P", "M", "L" } }, + { { "N", "D", "T", "M", "S", "R", "P", "L", "K" }, { "N", "D", "T", "M", "S", "R", "P", "L", "K" } }, + { { "N", "T", "S", "C", "K", "G", "E", "F", "Z", "V", "O", "M", "A" }, + { "N", "T", "S", "C", "G", "E", "F", "M", "A" } }, + { { "N", "E", "C", "T", "S", "G", "A", "V", "O", "P", "M", "L", "D", "B" }, + { "N", "E", "C", "T", "S", "G", "A", "P", "M", "L", "D" } }, + { { "H", "G" }, { "H" } }, { { "E", "P", "I", "O", "A" }, { "E" } }, + { { "E", "I", "S", "A", "U", "O" }, { "E", "S", "O" } }, + { { "E", "T", "I", "S", "L", "H" }, { "E", "T", "S", "H" } }, + { { "Y", "E", "O", "I", "S", "A" }, { "Y", "E", "S" } }, + { { "T", "S", "E", "I", "U", "O", "K", "H" }, { "T", "S", "E", "O", "H" } }, + { { "T", "N", "L", "I", "R", "K", "B", "G", "C" }, { "T", "N", "L", "R" } }, + { { "S", "D", "A", "T", "C", "R", "N", "M", "G", "V", "F" }, { "S", "D", "A", "T", "R", "N", "M" } }, + { { "R", "S", "V", "T", "U", "D" }, { "R", "T", "U", "D" } }, { { "R", "U" }, { "R", "U" } }, + { { "N", "L", "S", "T", "C", "G" }, { "N", "L", "S", "T", "C", "G" } }, { { "L", "O", "I" }, {} }, + { { "L", "Y", "I", "E", "F", "O", "A", "T", "S", "P", "D" }, { "L", "Y", "E", "F", "T", "S", "D" } }, + { { "L", "N", "T" }, { "L", "N", "T" } }, { { "L", "T", "R", "N", "M" }, { "L", "T", "R", "N", "M" } }, + { { "I", "E", "U", "O", "A" }, { "E", "O" } }, { { "E", "A", "O" }, { "E", "O" } }, + { { "E", "L", "I" }, { "E" } }, + { { "D", "S", "W", "R", "E", "Y", "V", "T", "L", "C", "A" }, { "D", "S", "W", "R", "E", "Y", "T", "L", "A" } }, + { { "A", "E", "I", "Y", "O" }, { "E", "Y" } }, + { { "T", "N", "R", "S", "C", "Y", "W", "I", "B" }, { "T", "N", "R", "S", "Y", "W" } }, + { { "T", "E", "S", "I" }, { "T", "E", "S" } }, + { { "S", "N", "R", "D", "P", "L", "I" }, { "S", "N", "R", "D", "P", "L" } }, + { { "S", "N", "T", "D", "F", "E", "C", "A", "V", "R" }, { "S", "N", "T", "D", "F", "E", "C", "A", "R" } }, + { { "R", "E", "C", "T", "L", "F", "S", "I", "G", "D", "A" }, { "R", "E", "T", "L", "S", "D", "A" } }, + { { "P", "E", "A" }, { "E" } }, + { { "O", "N", "D", "T", "S", "G", "C", "B", "V", "M", "A" }, { "N", "D", "T", "S", "G", "C", "M", "A" } }, + { { "N", "T", "S", "C", "Z", "O", "G", "F" }, { "N", "T", "S", "C", "G", "F" } }, + { { "N", "E", "S", "I", "A" }, { "N", "E", "S" } }, + { { "N", "M", "U", "L", "C", "R" }, { "N", "M", "U", "L", "R" } }, { { "E", "I" }, { "E" } }, + { { "E", "A", "I", "O", "U", "R" }, { "E", "O" } }, { { "E", "S", "P", "O", "B", "A", "I" }, { "E", "S" } }, + { { "E", "P", "I", "A", "S", "M" }, { "E", "S" } }, + { { "D", "N", "L", "S", "R", "E", "C", "T", "V", "A" }, { "D", "N", "L", "S", "R", "E", "T", "A" } }, + { { "T", "I", "E" }, { "T", "E" } }, { { "S", "R", "N", "L", "W", "T", "I" }, { "R", "N", "L", "W", "T" } }, + { { "R", "N", "G", "T" }, { "R", "N", "G", "T" } }, { { "P", "T", "I", "C", "A" }, { "T" } }, + { { "N" }, { "N" } }, { { "H", "T", "K", "E" }, { "H", "T", "K", "E" } }, + { { "E", "I", "Y", "V", "M", "D" }, { "E", "Y" } }, { { "E", "A", "O" }, { "E", "A" } }, + { { "E", "S", "T", "L", "I" }, { "E", "S", "T" } }, { { "E", "S", "L", "T", "R", "I" }, { "E", "S" } }, + { { "E", "P", "L" }, { "E" } }, { { "E", "O", "I", "A" }, { "E" } }, + { { "D", "L", "I", "O", "E", "U" }, { "D", "L", "E" } }, { { "Y", "T", "R", "N" }, { "Y", "T", "R", "N" } }, + { { "Y" }, { "Y" } }, { { "Y", "E" }, { "Y", "E" } }, + { { "W", "N", "O", "S", "C", "V", "U", "T", "R", "P", "G" }, { "W", "N", "O", "U", "T", "R", "P" } }, + { { "U", "T", "R", "O", "D", "A" }, { "U", "T", "R", "O", "D" } }, + { { "T", "E", "O", "I" }, { "T", "E", "O" } }, + { { "R", "E", "W", "L", "C", "A" }, { "R", "E", "W", "L", "A" } }, + { { "R", "N", "C", "E", "L", "G" }, { "R", "N", "C", "E", "L", "G" } }, + { { "R", "C", "P", "B", "M", "L", "A" }, { "R", "P", "M", "L" } }, + { { "N", "T", "S", "R", "D" }, { "N", "T", "S", "R", "D" } }, + { { "L", "O", "A", "T", "R", "E" }, { "T", "E" } }, { { "L", "T", "R" }, { "L", "T", "R" } }, + { { "K", "D", "L", "T", "R", "N", "M" }, { "K", "D", "L", "T", "R", "N", "M" } }, + { { "I", "H", "A", "E", "Y", "U", "S" }, { "H", "A", "E", "Y", "S" } }, + { { "I", "M", "Y", "N", "L" }, { "M", "Y", "N", "L" } }, { { "E", "I", "O", "A" }, { "E", "A" } }, + { { "E", "I" }, { "E" } }, { { "E" }, { "E" } }, { { "E" }, { "E" } }, + { { "D", "N", "T", "S", "R", "E" }, { "D", "N", "T", "S", "R", "E" } }, + { { "C", "R", "M", "I" }, { "R", "M" } }, + { { "A", "T", "E", "S", "P", "N" }, { "A", "T", "E", "S", "P", "N" } }, { { "U" }, {} }, + { { "T", "F" }, { "T", "F" } }, { { "T", "I", "H", "E", "Y", "W", "S", "A" }, { "H", "E", "Y", "S", "A" } }, + { { "S", "E" }, { "S" } }, { { "S", "N", "L", "C" }, { "S", "N", "L" } }, { { "S", "N" }, { "S", "N" } }, + { { "O", "E", "I" }, { "E" } }, { { "O", "E" }, { "O", "E" } }, { { "N", "V", "O", "C" }, { "N", "C" } }, + { { "N" }, { "N" } }, { { "N", "D", "S", "C", "T", "O", "L", "E" }, { "N", "D", "S", "C", "T", "L", "E" } }, + { { "L", "R", "T", "S" }, { "L", "R", "T", "S" } }, { { "L", "R", "N" }, { "L", "R", "N" } }, + { { "I", "U", "E" }, { "E" } }, { { "I", "E", "A" }, { "E" } }, { { "I", "H", "E" }, { "H", "E" } }, + { { "H", "E" }, { "H", "E" } }, { { "F", "T" }, { "F", "T" } }, { { "E", "A", "U", "O" }, { "E" } }, + { { "E" }, { "E" } }, { { "E", "A" }, { "E" } }, { { "E", "O", "R", "L" }, { "E" } }, { { "E", "A" }, { "E" } }, + { { "C", "S", "R", "A" }, { "S", "R" } }, { { "A", "S", "I", "E" }, { "S", "E" } }, + { { "A", "S", "D" }, { "A", "S", "D" } }, { { "Y", "D" }, { "Y", "D" } }, + { { "W", "N", "M", "E" }, { "W", "N", "M" } }, { { "T", "S" }, { "T", "S" } }, + { { "T", "O", "E", "A" }, { "T", "E" } }, { { "S", "C", "R", "N", "L" }, { "S", "R", "N", "L" } }, + { { "S" }, { "S" } }, { { "S", "E", "I" }, { "S", "E" } }, { { "S", "N", "C" }, { "S", "N" } }, + { { "R", "I" }, {} }, { { "O", "I", "E", "A" }, { "E", "A" } }, + { { "O", "F", "U", "T", "E" }, { "F", "T", "E" } }, { { "O", "E" }, { "O", "E" } }, { { "O" }, { "O" } }, + { { "N", "I", "T", "R" }, { "N", "T", "R" } }, { { "N", "T", "R", "E", "C" }, { "N", "T", "R", "E", "C" } }, + { { "N" }, { "N" } }, { { "L", "T", "N" }, { "L", "T", "N" } }, { { "L", "I", "E" }, { "E" } }, { { "L" }, {} }, + { { "L", "T", "R", "N" }, { "L", "T", "R", "N" } }, { { "K", "I", "E", "C", "A" }, { "K", "E" } }, + { { "I", "F", "E", "T" }, { "F", "E", "T" } }, { { "I", "E", "M", "A" }, { "E", "A" } }, + { { "I", "E", "Y", "O" }, { "E", "Y" } }, { { "H", "R", "O", "I", "A" }, { "H" } }, { { "H" }, { "H" } }, + { { "E" }, { "E" } }, { { "E" }, { "E" } }, { { "E", "O", "I", "A" }, { "E", "A" } }, + { { "E", "U", "I" }, { "E" } }, { { "E", "O", "I" }, { "E", "O" } }, { { "E", "Y", "U" }, { "E", "Y" } }, + { { "E", "I" }, { "E" } }, { { "E", "I" }, { "E" } }, { { "E" }, { "E" } }, { { "C" }, {} }, + { { "B", "E" }, { "E" } }, { { "A", "R", "I", "E" }, { "E" } }, { { "Y", "E" }, { "Y", "E" } }, + { { "Y", "O", "I", "E" }, { "Y", "O", "E" } }, { { "Y", "A" }, { "Y" } }, { { "V", "T", "O" }, { "T", "O" } }, + { { "U", "O", "E" }, { "E" } }, { { "T", "S", "M", "E", "D" }, { "T", "S", "M", "E" } }, + { { "T", "R", "D" }, { "T", "R", "D" } }, { { "T", "R", "L", "B" }, { "T", "R", "L" } }, + { { "T", "R", "L" }, { "T", "R", "L" } }, { { "T" }, { "T" } }, { { "T" }, { "T" } }, + { { "S", "R", "N", "M" }, { "S", "R", "N", "M" } }, { { "S" }, { "S" } }, { { "S" }, { "S" } }, + { { "S" }, { "S" } }, { { "R", "P" }, { "R", "P" } }, { { "P" }, {} }, { { "P", "O" }, {} }, + { { "O", "E" }, { "E" } }, { { "O" }, {} }, { { "O" }, {} }, { { "O" }, {} }, { { "N" }, {} }, + { { "M" }, { "M" } }, { { "M" }, { "M" } }, { { "L" }, {} }, { { "L" }, { "L" } }, { { "I" }, {} }, + { { "I" }, {} }, { { "I", "E" }, { "E" } }, { { "I" }, {} }, { { "I", "E" }, { "E" } }, { { "I" }, {} }, + { { "H" }, {} }, { { "H", "E" }, { "H", "E" } }, { { "H" }, { "H" } }, { { "F" }, { "F" } }, { { "E" }, {} }, + { { "E" }, { "E" } }, { { "E" }, {} }, { { "E" }, { "E" } }, { { "E", "A" }, { "E" } }, { { "E" }, { "E" } }, + { { "E" }, { "E" } }, { { "E" }, { "E" } }, { { "E" }, { "E" } }, { { "E" }, { "E" } }, { { "E" }, { "E" } }, + { { "E" }, { "E" } }, { { "E" }, { "E" } }, { { "E" }, { "E" } }, { { "E" }, { "E" } }, { { "E" }, { "E" } }, + { { "E" }, { "E" } }, { { "D" }, { "D" } }, { { "A" }, {} }, { { "A" }, {} } }; + // ----------------------------------------------------------------------------------------- + randomWord = startBiGram[indexGenerator(startBiGram.length)]; + int flag = 0; + int count = 0; + String previousWord; + while (randomWord.length() != wordLength) { + previousWord = randomWord; + randomWord = addCharacter(startBiGram, wordLength, randomWord, lookupBiGram, nextCharLookup, flag); + if (previousWord.equals(randomWord)) { + count++; + } else { + flag = 0; + } + if (count == 5) { + flag = 1; + count++; + } else if (count == 20) { + randomWord = startBiGram[indexGenerator(startBiGram.length)]; + count = 0; + } + } + return randomWord; + } + + private static String addCharacter(String[] startBiGram, int desiredLength, String currentWord, String[] lookupBiGram, + String[][][] nextCharLookup, int flag) { + int mainIndex = getLookupIndex(currentWord, lookupBiGram); + int type = 0; + if (currentWord.length() == (desiredLength - 1)) { + type = 1; + } + while (mainIndex < 0 || mainIndex > 263 || nextCharLookup[mainIndex][type].length <= 0) { + if (currentWord.length() == 2) { + return startBiGram[indexGenerator(startBiGram.length)]; + } + if (flag == 1) { + currentWord = backtrack(currentWord, 2); + flag = 0; + } else { + currentWord = backtrack(currentWord, 1); + } + mainIndex = getLookupIndex(currentWord, lookupBiGram); + if (type == 1) { + type = 0; + } + } + String updatedWord = currentWord + getNextCharacter(type, mainIndex, nextCharLookup); + return updatedWord; + } + + private static int indexGenerator(int arrayLength) { + int theIndex; + Random generator = new Random(); + theIndex = generator.nextInt(arrayLength); + return theIndex; + } + + private static String getNextCharacter(int type, int mainIndex, String[][][] theCharacterVault) { + String nextChar; + int i = indexGenerator(theCharacterVault[mainIndex][type].length); + nextChar = theCharacterVault[mainIndex][type][i]; + return nextChar; + } + + private static String backtrack(String theWord, int numberChars) { + theWord = theWord.substring(0, theWord.length() - numberChars); + return theWord; + } + + private static int getLookupIndex(String theWord, String[] lookupArray) { + String lookupCharacters = theWord.substring(theWord.length() - 2); + int lookupIndex = Arrays.asList(lookupArray).indexOf(lookupCharacters); + return lookupIndex; + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/utils/WordLengthException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/utils/WordLengthException.java new file mode 100644 index 00000000000..a54b1779869 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/utils/WordLengthException.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.appcatalog.utils; + +public class WordLengthException extends Exception { + /** + * + */ + private static final long serialVersionUID = -217354336577036950L; + + public WordLengthException(String message){ + super(message); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java index fc65a63cc19..89234752c28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java @@ -24,6 +24,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.appcatalog.controller.AppCatalogController; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; @@ -136,7 +137,8 @@ private void startWebApp() throws IOException { String apiPackages = ApiServer.class.getPackage().getName() + SEP + GenericExceptionHandler.class.getPackage().getName() + SEP - + YarnJacksonJaxbJsonProvider.class.getPackage().getName(); + + YarnJacksonJaxbJsonProvider.class.getPackage().getName() + SEP + + AppCatalogController.class.getPackage().getName(); apiServer.addJerseyResourcePackage(apiPackages, "/*"); try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/application/EmbeddedSolrServerFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/application/EmbeddedSolrServerFactory.java new file mode 100644 index 00000000000..e6d90447d97 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/application/EmbeddedSolrServerFactory.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.appcatalog.application; + +import org.apache.commons.io.FileUtils; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.core.NodeConfig; +import org.apache.solr.core.SolrResourceLoader; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class EmbeddedSolrServerFactory { + + /** + * Cleans the given solrHome directory and creates a new EmbeddedSolrServer. + * + * @param solrHome the Solr home directory to use + * @param configSetHome the directory containing config sets + * @param coreName the name of the core, must have a matching directory in configHome + * + * @return an EmbeddedSolrServer with a core created for the given coreName + * @throws IOException + */ + public static SolrClient create(final String solrHome, final String configSetHome, final String coreName) + throws IOException, SolrServerException { + return create(solrHome, configSetHome, coreName, true); + } + + /** + * @param solrHome the Solr home directory to use + * @param configSetHome the directory containing config sets + * @param coreName the name of the core, must have a matching directory in configHome + * @param cleanSolrHome if true the directory for solrHome will be deleted and re-created if it already exists + * + * @return an EmbeddedSolrServer with a core created for the given coreName + * @throws IOException + */ + public static SolrClient create(final String solrHome, final String configSetHome, final String coreName, final boolean cleanSolrHome) + throws IOException, SolrServerException { + + final File solrHomeDir = new File(solrHome); + if (solrHomeDir.exists()) { + FileUtils.deleteDirectory(solrHomeDir); + solrHomeDir.mkdirs(); + } else { + solrHomeDir.mkdirs(); + } + + final SolrResourceLoader loader = new SolrResourceLoader(solrHomeDir.toPath()); + final Path configSetPath = Paths.get(configSetHome).toAbsolutePath(); + + final NodeConfig config = new NodeConfig.NodeConfigBuilder("embeddedSolrServerNode", loader) + .setConfigSetBaseDirectory(configSetPath.toString()) + .build(); + + final EmbeddedSolrServer embeddedSolrServer = new EmbeddedSolrServer(config, coreName); + + final CoreAdminRequest.Create createRequest = new CoreAdminRequest.Create(); + createRequest.setCoreName(coreName); + createRequest.setConfigSet(coreName); + embeddedSolrServer.request(createRequest); + + return embeddedSolrServer; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestYarnSolrClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestYarnSolrClient.java new file mode 100644 index 00000000000..d7913564697 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestYarnSolrClient.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.application; + +import org.apache.hadoop.yarn.appcatalog.application.EmbeddedSolrServerFactory; +import org.apache.hadoop.yarn.appcatalog.application.YarnSolrClient; +import org.apache.hadoop.yarn.appcatalog.model.AppCatalogEntry; +import org.apache.hadoop.yarn.appcatalog.model.Application; +import org.apache.solr.client.solrj.SolrClient; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.powermock.api.mockito.PowerMockito; +import static org.powermock.api.mockito.PowerMockito.when; +import static org.powermock.api.support.membermodification.MemberMatcher.method; + +import static org.junit.Assert.*; + +import java.util.List; + +public class TestYarnSolrClient { + + static final String CONFIGSET_DIR = "src/test/resources/configsets"; + static SolrClient solrClient; + static YarnSolrClient spy; + + + @Before + public void setup() throws Exception { + String targetLocation = EmbeddedSolrServerFactory.class + .getProtectionDomain().getCodeSource().getLocation().getFile() + "/.."; + + String solrHome = targetLocation + "/solr"; + solrClient = EmbeddedSolrServerFactory.create(solrHome, CONFIGSET_DIR, "exampleCollection"); + spy = PowerMockito.spy(new YarnSolrClient()); + when(spy, method(YarnSolrClient.class, "getSolrClient")).withNoArguments().thenReturn(solrClient); + } + + @After + public void teardown() throws Exception { + try { + solrClient.close(); + } catch (Exception e) { + } + } + + @Test + public void testRegister() throws Exception { + Application example = new Application(); + example.setOrganization("jenkins-ci.org"); + example.setName("jenkins"); + example.setDescription("World leading open source automation system."); + example.setIcon("/css/img/feather.png"); + spy.register(example); + List apps = spy.getRecommendedApps(); + assertEquals(1, apps.size()); + } + + @Test + public void testSearch() throws Exception { + Application example = new Application(); + example.setOrganization("jenkins-ci.org"); + example.setName("jenkins"); + example.setDescription("World leading open source automation system."); + example.setIcon("/css/img/feather.png"); + spy.register(example); + List results = spy.search("name_s:jenkins"); + int expected = 1; + int actual = results.size(); + assertEquals(expected, actual); + } + + @Test + public void testNotFoundSearch() throws Exception { + Application example = new Application(); + example.setOrganization("jenkins-ci.org"); + example.setName("jenkins"); + example.setDescription("World leading open source automation system."); + example.setIcon("/css/img/feather.png"); + spy.register(example); + List results = spy.search("name_s:abc"); + int expected = 0; + int actual = results.size(); + assertEquals(expected, actual); + } + + @Test + public void testGetRecommendedApps() throws Exception { + List expected = spy.getRecommendedApps(); + List actual = spy.getRecommendedApps(); + assertEquals(expected, actual); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppCatalogController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppCatalogController.java new file mode 100644 index 00000000000..d127fb306f2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppCatalogController.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.controller; + +import org.apache.hadoop.yarn.appcatalog.controller.AppCatalogController; +import org.apache.hadoop.yarn.appcatalog.model.AppCatalogEntry; +import org.apache.hadoop.yarn.appcatalog.model.Application; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import javax.ws.rs.Path; +import javax.ws.rs.core.Response; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +import java.util.ArrayList; +import java.util.List; + +public class TestAppCatalogController { + + private AppCatalogController controller; + + @Before + public void setUp() throws Exception { + this.controller = new AppCatalogController(); + + } + + @Test + public void testGetRecommended() throws Exception { + AppCatalogController ac = Mockito.mock(AppCatalogController.class); + List actual = new ArrayList(); + when(ac.get()).thenReturn(actual); + final List result = ac.get(); + assertEquals(result, actual); + } + + @Test + public void testSearch() throws Exception { + String keyword = "jenkins"; + AppCatalogController ac = Mockito.mock(AppCatalogController.class); + List expected = new ArrayList(); + when(ac.search(keyword)).thenReturn(expected); + final List actual = ac.search(keyword); + assertEquals(expected, actual); + } + + @Test + public void testRegister() throws Exception { + AppCatalogController ac = Mockito.mock(AppCatalogController.class); + Application app = new Application(); + app.setName("jenkins"); + app.setOrganization("jenkins.org"); + app.setDescription("This is a description"); + app.setIcon("/css/img/feather.png"); + Response expected = Response.ok().build(); + when(ac.register(app)).thenReturn(Response.ok().build()); + final Response actual = ac.register(app); + assertEquals(expected.getStatus(), actual.getStatus()); + } + + @Test + public void testPathAnnotation() throws Exception { + assertNotNull(this.controller.getClass() + .getAnnotations()); + assertThat("The controller has the annotation Path", this.controller.getClass() + .isAnnotationPresent(Path.class)); + + final Path path = this.controller.getClass() + .getAnnotation(Path.class); + assertThat("The path is /ws/v2/app_catalog", path.value(), is("/ws/v2/app_catalog")); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppDetailsController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppDetailsController.java new file mode 100644 index 00000000000..dd2444ba52e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppDetailsController.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.controller; + +import org.apache.hadoop.yarn.appcatalog.controller.AppDetailsController; +import org.apache.hadoop.yarn.appcatalog.model.AppEntry; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.Container; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import javax.ws.rs.Path; +import javax.ws.rs.core.Response; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +import java.util.ArrayList; +import java.util.List; + +public class TestAppDetailsController { + + private AppDetailsController controller; + + @Before + public void setUp() throws Exception { + this.controller = new AppDetailsController(); + + } + + @Test + public void testGetDetails() throws Exception { + String id = "application 1"; + AppDetailsController ac = Mockito.mock(AppDetailsController.class); + + AppEntry actual = new AppEntry(); + actual.setName(id); + when(ac.getDetails(id)).thenReturn(actual); + final AppEntry result = ac.getDetails(id); + assertEquals(result, actual); + } + + @Test + public void testGetStatus() throws Exception { + String id = "application 1"; + AppDetailsController ac = Mockito.mock(AppDetailsController.class); + + Service yarnfile = new Service(); + Component comp = new Component(); + Container c = new Container(); + c.setId("container-1"); + List containers = new ArrayList(); + containers.add(c); + comp.setContainers(containers); + yarnfile.addComponent(comp); + AppEntry actual = new AppEntry(); + actual.setName(id); + actual.setYarnfile(yarnfile); + when(ac.getStatus(id)).thenReturn(actual); + final AppEntry result = ac.getStatus(id); + assertEquals(result, actual); + } + + @Test + public void testStopApp() throws Exception { + String id = "application 1"; + AppDetailsController ac = Mockito.mock(AppDetailsController.class); + + Service yarnfile = new Service(); + Component comp = new Component(); + Container c = new Container(); + c.setId("container-1"); + List containers = new ArrayList(); + containers.add(c); + comp.setContainers(containers); + yarnfile.addComponent(comp); + Response expected = Response.ok().build(); + when(ac.stopApp(id)).thenReturn(Response.ok().build()); + final Response actual = ac.stopApp(id); + assertEquals(expected.getStatus(), actual.getStatus()); + } + + @Test + public void testRestartApp() throws Exception { + String id = "application 1"; + AppDetailsController ac = Mockito.mock(AppDetailsController.class); + + Service yarnfile = new Service(); + Component comp = new Component(); + Container c = new Container(); + c.setId("container-1"); + List containers = new ArrayList(); + containers.add(c); + comp.setContainers(containers); + yarnfile.addComponent(comp); + Response expected = Response.ok().build(); + when(ac.restartApp(id)).thenReturn(Response.ok().build()); + final Response actual = ac.restartApp(id); + assertEquals(expected.getStatus(), actual.getStatus()); + } + + @Test + public void testPathAnnotation() throws Exception { + assertNotNull(this.controller.getClass() + .getAnnotations()); + assertThat("The controller has the annotation Path", this.controller.getClass() + .isAnnotationPresent(Path.class)); + + final Path path = this.controller.getClass() + .getAnnotation(Path.class); + assertThat("The path is /ws/v2/app_details", path.value(), is("/ws/v2/app_details")); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppListController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppListController.java new file mode 100644 index 00000000000..151e53511cd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppListController.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.controller; + +import org.apache.hadoop.yarn.appcatalog.controller.AppListController; +import org.apache.hadoop.yarn.appcatalog.model.AppEntry; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import javax.ws.rs.Path; +import javax.ws.rs.core.Response; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +import java.util.ArrayList; +import java.util.List; + +public class TestAppListController { + + private AppListController controller; + + @Before + public void setUp() throws Exception { + this.controller = new AppListController(); + + } + + @Test + public void testGetList() throws Exception { + AppListController ac = Mockito.mock(AppListController.class); + + List actual = new ArrayList(); + when(ac.getList()).thenReturn(actual); + final List result = ac.getList(); + assertEquals(result, actual); + } + + @Test + public void testDelete() throws Exception { + String id = "application 1"; + AppListController ac = Mockito.mock(AppListController.class); + + Response expected = Response.ok().build(); + when(ac.delete(id, id)).thenReturn(Response.ok().build()); + final Response actual = ac.delete(id, id); + assertEquals(expected.getStatus(), actual.getStatus()); + } + + @Test + public void testDeploy() throws Exception { + String id = "application 1"; + AppListController ac = Mockito.mock(AppListController.class); + + Response expected = Response.ok().build(); + when(ac.deploy(id)).thenReturn(Response.ok().build()); + final Response actual = ac.deploy(id); + assertEquals(expected.getStatus(), actual.getStatus()); + } + + @Test + public void testPathAnnotation() throws Exception { + assertNotNull(this.controller.getClass() + .getAnnotations()); + assertThat("The controller has the annotation Path", this.controller.getClass() + .isAnnotationPresent(Path.class)); + + final Path path = this.controller.getClass() + .getAnnotation(Path.class); + assertThat("The path is /ws/v2/app_list", path.value(), is("/ws/v2/app_list")); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt new file mode 100644 index 00000000000..2c164c0b2a1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +# Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +such +that +the +their +then +there +these +they +this +to +was +will +with diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/params.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/params.json new file mode 100644 index 00000000000..06114ef2577 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/params.json @@ -0,0 +1,20 @@ +{"params":{ + "query":{ + "defType":"edismax", + "q.alt":"*:*", + "rows":"10", + "fl":"*,score", + "":{"v":0} + }, + "facets":{ + "facet":"on", + "facet.mincount": "1", + "":{"v":0} + }, + "velocity":{ + "wt": "velocity", + "v.template":"browse", + "v.layout": "layout", + "":{"v":0} + } +}} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/protwords.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/protwords.txt new file mode 100644 index 00000000000..1dfc0abecbf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/schema.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/schema.xml new file mode 100644 index 00000000000..20acbc92b46 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/schema.xml @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml new file mode 100644 index 00000000000..392feec28aa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml @@ -0,0 +1,36 @@ + + + + 6.2.1 + + ${solr.data.dir:} + + + + + single + + + + + + + + + + \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/stopwords.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/stopwords.txt new file mode 100644 index 00000000000..ae1e83eeb3d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/stopwords.txt @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/synonyms.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/synonyms.txt new file mode 100644 index 00000000000..7f72128303b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/synonyms.txt @@ -0,0 +1,29 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaafoo => aaabar +bbbfoo => bbbfoo bbbbar +cccfoo => cccbar cccbaz +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/log4j.properties new file mode 100644 index 00000000000..81a3f6ad5d2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/log4j.properties @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# log4j configuration used during build and unit tests + +log4j.rootLogger=info,stdout +log4j.threshold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index fb07edc908f..b3d8edcfad3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -22,6 +22,7 @@ 3.1.0-SNAPSHOT hadoop-yarn-services-core + 3.1.0-SNAPSHOT jar Apache Hadoop YARN Services Core diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml index 716fdb70cf0..3b5eaf8a27c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml @@ -24,6 +24,7 @@ 4.0.0 hadoop-yarn-services Apache Hadoop YARN Services + 3.1.0-SNAPSHOT pom diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index 1b3c5f05620..b280d1df57e 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -86,6 +86,10 @@ org.apache.hadoop hadoop-yarn-services-core + + org.apache.hadoop + hadoop-yarn-services-api +