commit 872b6381a12ef2128521b5db7e9ea5d0f3eac811 Author: Eric Yang Date: Wed Sep 13 16:48:19 2017 -0700 YARN-7129. Implemented REST for registering YARN application in application catalog. diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 31ac611f65d..29194f168d4 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -152,12 +152,19 @@ ENV MAVEN_OPTS -Xms256m -Xmx512m ### # Install node js tools for web UI frameowkr ### -RUN apt-get -y install nodejs && \ - ln -s /usr/bin/nodejs /usr/bin/node && \ +RUN mkdir -p /opt/nodejs && \ + curl -L -s -S \ + https://nodejs.org/download/release/v6.11.2/node-v6.11.2-linux-x64.tar.gz \ + -o /opt/node-v6.11.2-linux-x64.tar.gz && \ + tar xzf /opt/node-v6.11.2-linux-x64.tar.gz --strip-components 1 -C /opt/nodejs && \ + ln -s /opt/nodejs/bin/node /usr/bin/nodejs && \ + ln -s /opt/nodejs/bin/node /usr/bin/node && \ apt-get -y install npm && \ npm install npm@latest -g && \ npm install -g bower && \ - npm install -g ember-cli + npm install -g ember-cli && \ + npm install -g phantomjs && \ + npm install -g apidoc ### # Everything past this point is either not needed for testing or breaks Yetus. diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index c3366a5cc1b..01ce06507af 100755 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -431,6 +431,12 @@ org.apache.hadoop + hadoop-yarn-services-api + ${project.version} + + + + org.apache.hadoop hadoop-mapreduce-client-jobclient ${project.version} test-jar @@ -976,7 +982,7 @@ org.mockito mockito-all - 1.8.5 + 1.9.5 org.mock-server diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml index 74d9681bee4..f94ebdec747 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml @@ -23,9 +23,15 @@ hadoop-yarn-services-api Apache Hadoop YARN Services API + 3.1.0-SNAPSHOT jar Hadoop YARN REST APIs for services + + 1.5.6 + 6.2.0 + + @@ -42,6 +48,61 @@ + + + org.apache.rat + apache-rat-plugin + + + src/test/resources/configsets/exampleCollection/conf/params.json + + + + + org.apache.maven.plugins + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + + + api + X + + + apiGroup + X + + + apiName + X + + + apiParam + X + + + apiParamExample + X + + + apiSuccess + X + + + apiSuccessExample + X + + + apiError + X + + + apiErrorExample + X + + + + org.apache.maven.plugins maven-jar-plugin @@ -74,9 +135,32 @@ ${java.home} + + file:${project.build.testOutputDirectory}/log4j.properties + + + org.codehaus.mojo + exec-maven-plugin + 1.3.2 + + + apidocs + package + + exec + + + ${project.build.directory}/apidocs.log + apidoc + -i src/main/java -o ${project.build.directory}/site/apidocs + true + + + + @@ -116,5 +200,210 @@ javax.ws.rs jsr311-api + + + org.apache.hadoop + hadoop-common + test-jar + test + + + + junit + junit + 4.11 + test + + + org.mockito + mockito-all + 1.9.5 + test + + + org.powermock + powermock-module-junit4 + ${powermock.version} + + + org.javassist + javassist + + + test + + + org.powermock + powermock-api-mockito + ${powermock.version} + + + org.hamcrest + hamcrest-core + + + org.objenesis + objenesis + + + test + + + com.fasterxml.jackson.module + jackson-module-jaxb-annotations + 2.7.8 + + + + commons-io + commons-io + 2.5 + test + + + + org.glassfish.jersey.media + jersey-media-json-jackson + 2.12 + + + org.eclipse.jetty + jetty-xml + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-base + + + test + + + + com.sun.jersey + jersey-client + 1.19 + provided + + + + com.sun.jersey + jersey-server + 1.19 + provided + + + + asm + asm-commons + 3.3.1 + + + + org.apache.solr + solr-solrj + ${solr.version} + + + org.eclipse.jetty + jetty-xml + + + org.slf4j + jcl-over-slf4j + + + + + org.apache.solr + solr-core + ${solr.version} + + + org.eclipse.jetty + jetty-http + + + org.eclipse.jetty + jetty-io + + + org.eclipse.jetty + jetty-security + + + org.eclipse.jetty + jetty-xml + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + slf4j-api + + + test + + + + org.apache.solr + solr-test-framework + ${solr.version} + + + org.eclipse.jetty + jetty-http + + + org.eclipse.jetty + jetty-io + + + org.eclipse.jetty + jetty-security + + + org.eclipse.jetty + jetty-xml + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + slf4j-api + + + test + + + + com.fasterxml.jackson.core + jackson-core + 2.7.8 + provided + + + + com.fasterxml.jackson.core + jackson-annotations + 2.7.8 + provided + + + + + + disable-java8-doclint + + [1.8,) + + + -Xdoclint:none + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/Appcatalog.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/Appcatalog.java new file mode 100644 index 00000000000..df56ee010c6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/Appcatalog.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.application; + +import javax.ws.rs.ApplicationPath; +import javax.ws.rs.core.Application; + +import org.apache.hadoop.yarn.appcatalog.controller.AppDetailsController; + +import java.util.Set; + +/** + * Application configuration for application catalog. + * + */ +@ApplicationPath("service") +public class Appcatalog extends Application { + + @Override + public Set> getClasses() { + final Set> resources = new java.util.HashSet<>(); + // following code can be used to customize Jersey 2.0 JSON provider: + try { + final Class jsonProvider = Class. + forName("org.glassfish.jersey.jackson.JacksonFeature"); + resources.add(jsonProvider); + } catch (final ClassNotFoundException ex) { + ex.printStackTrace(); + } + addRestResourceClasses(resources); + return resources; + } + + /** + * Add your own resources here. + */ + private void addRestResourceClasses(final Set> resources) { + resources.add(AppDetailsController.class); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnClient.java new file mode 100644 index 00000000000..929616aedd6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnClient.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.application; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.appcatalog.model.AppEntry; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.client.ServiceClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.sun.jersey.api.client.UniformInterfaceException; + +/** + * A singleton client to access Yarn native service api. + * + */ +public class YarnClient { + + private static final Logger LOG = + LoggerFactory.getLogger(YarnClient.class); + + private static Configuration yarnCONFIG = new YarnConfiguration(); + private static ServiceClient serviceClient; + + protected YarnClient() { + } + + static { + init(); + } + + private static void init() { + serviceClient = new ServiceClient(); + serviceClient.init(yarnCONFIG); + serviceClient.start(); + } + + public static void createApp(Service app) { + try { + serviceClient.actionCreate(app); + } catch (IOException | YarnException e) { + LOG.error("Error in deploying application: ", e); + throw new RuntimeException("Failed to create application: " + + app.getName() + " error: " + e.getMessage()); + } + } + + public static void deleteApp(String appName) { + boolean destroy = true; + try { + serviceClient.actionStop(appName, destroy); + if (destroy) { + serviceClient.actionDestroy(appName); + LOG.info("Successfully deleted service {}", appName); + } else { + LOG.info("Successfully stopped service {}", appName); + } + } catch (Exception e) { + LOG.error("Error in deleting application: ", e); + throw new RuntimeException("Failed to delete application: " + + appName + " error: " + e.getMessage()); + } + } + + public static void restartApp(Service app) throws JsonProcessingException { + String appName = app.getName(); + try { + int result = serviceClient.actionStart(appName); + if (result != 0) { + throw new RuntimeException("Failed to restart application: " + appName); + } + } catch (IOException | YarnException | UniformInterfaceException e) { + LOG.error("Error in stopping application: ", e); + } + + } + + public static void stopApp(Service app) throws JsonProcessingException { + String appName = app.getName(); + try { + serviceClient.actionStop(appName, true); + } catch (IOException | YarnException | UniformInterfaceException e) { + LOG.error("Error in stopping application: ", e); + } + } + + public static void getStatus(AppEntry entry) { + try { + Service app = serviceClient.getStatus(entry.getName()); + entry.setYarnfile(app); + } catch (UniformInterfaceException | IOException | YarnException e) { + LOG.error("Error in fetching application status: ", e); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnSolrClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnSolrClient.java new file mode 100644 index 00000000000..ad44743f27f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnSolrClient.java @@ -0,0 +1,328 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.application; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.appcatalog.model.AppEntry; +import org.apache.hadoop.yarn.appcatalog.model.AppCatalogEntry; +import org.apache.hadoop.yarn.appcatalog.model.Application; +import org.apache.hadoop.yarn.appcatalog.utils.RandomWord; +import org.apache.hadoop.yarn.appcatalog.utils.WordLengthException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrInputDocument; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * YARN application catalog solr client. + * + */ +public class YarnSolrClient { + + private static final Log LOG = LogFactory.getLog(YarnSolrClient.class); + private static String solrURL; + + public YarnSolrClient() { + } + + static { + init(); + } + + public static void init() { + YarnConfiguration conf = new YarnConfiguration(); + solrURL = "http://" + conf.get("yarn.apiserver.webapp.address", + "localhost:8983/solr/appcatalog"); + } + + public SolrClient getSolrClient() { + return new HttpSolrClient.Builder(solrURL).build(); + } + + public List getRecommendedApps() { + List apps = new ArrayList(); + SolrClient solr = getSolrClient(); + SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + query.setFilterQueries("type_s:AppCatalogEntry"); + query.setRows(40); + QueryResponse response; + try { + response = solr.query(query); + Iterator list = response.getResults().listIterator(); + while (list.hasNext()) { + SolrDocument d = list.next(); + AppCatalogEntry entry = new AppCatalogEntry(); + entry.setId(d.get("id").toString()); + entry.setOrg(d.get("org_s").toString()); + entry.setName(d.get("name_s").toString()); + entry.setDesc(d.get("desc_s").toString()); + if (d.get("icon_s") != null) { + entry.setIcon(d.get("icon_s").toString()); + } + entry.setLike(Integer.parseInt(d.get("like_i").toString())); + entry.setDownload(Integer.parseInt(d.get("download_i").toString())); + apps.add(entry); + } + } catch (SolrServerException | IOException e) { + LOG.error("Error getting a list of recommended applications: ", e); + } + return apps; + } + + public List search(String keyword) { + List apps = new ArrayList(); + SolrClient solr = getSolrClient(); + SolrQuery query = new SolrQuery(); + if (keyword.length() == 0) { + query.setQuery("*:*"); + query.setFilterQueries("type_s:AppCatalogEntry"); + } else { + query.setQuery(keyword); + query.setFilterQueries("type_s:AppCatalogEntry"); + } + query.setRows(40); + QueryResponse response; + try { + response = solr.query(query); + Iterator list = response.getResults().listIterator(); + while (list.hasNext()) { + SolrDocument d = list.next(); + AppCatalogEntry entry = new AppCatalogEntry(); + entry.setId(d.get("id").toString()); + entry.setOrg(d.get("org_s").toString()); + entry.setName(d.get("name_s").toString()); + entry.setDesc(d.get("desc_s").toString()); + entry.setLike(Integer.parseInt(d.get("like_i").toString())); + entry.setDownload(Integer.parseInt(d.get("download_i").toString())); + apps.add(entry); + } + } catch (SolrServerException | IOException e) { + LOG.error("Error in searching for applications: ", e); + } + return apps; + } + + public List listAppEntries() { + List list = new ArrayList(); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + SolrClient solr = getSolrClient(); + SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + query.setFilterQueries("type_s:AppEntry"); + query.setRows(40); + QueryResponse response; + try { + response = solr.query(query); + Iterator appList = response.getResults().listIterator(); + while (appList.hasNext()) { + SolrDocument d = appList.next(); + AppEntry entry = new AppEntry(); + entry.setId(d.get("id").toString()); + entry.setName(d.get("name_s").toString()); + entry.setApp(d.get("app_s").toString()); + entry.setYarnfile(mapper.readValue(d.get("yarnfile_s").toString(), + org.apache.hadoop.yarn.service.api.records.Service.class)); + list.add(entry); + } + } catch (SolrServerException | IOException e) { + LOG.error("Error in listing deployed applications: ", e); + } + return list; + } + + public AppEntry findAppEntry(String id) { + AppEntry entry = new AppEntry(); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + SolrClient solr = getSolrClient(); + SolrQuery query = new SolrQuery(); + query.setQuery("id:" + id); + query.setFilterQueries("type_s:AppEntry"); + query.setRows(1); + + QueryResponse response; + try { + response = solr.query(query); + Iterator appList = response.getResults().listIterator(); + while (appList.hasNext()) { + SolrDocument d = appList.next(); + entry.setId(d.get("id").toString()); + entry.setApp(d.get("app_s").toString()); + entry.setName(d.get("name_s").toString()); + entry.setYarnfile(mapper.readValue(d.get("yarnfile_s").toString(), + org.apache.hadoop.yarn.service.api.records.Service.class)); + } + } catch (SolrServerException | IOException e) { + LOG.error("Error in finding deployed application: " + id, e); + } + return entry; + } + + public org.apache.hadoop.yarn.service.api.records.Service deployApp(String id) + throws SolrServerException, IOException { + long download = 0; + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + Collection docs = new HashSet(); + SolrClient solr = getSolrClient(); + // Find application information from AppCatalog + String name; + try { + Random r = new Random(); + int low = 3; + int high = 10; + int seed = r.nextInt(high - low) + low; + int seed2 = r.nextInt(high - low) + low; + name = RandomWord.getNewWord(seed).toLowerCase() + "-" + + RandomWord.getNewWord(seed2).toLowerCase(); + } catch (WordLengthException e) { + name = "c" + java.util.UUID.randomUUID().toString().substring(0, 11); + } + org.apache.hadoop.yarn.service.api.records.Service yarnApp = null; + SolrQuery query = new SolrQuery(); + query.setQuery("id:" + id); + query.setFilterQueries("type_s:AppCatalogEntry"); + query.setRows(1); + QueryResponse response = solr.query(query); + Iterator appList = response.getResults().listIterator(); + AppCatalogEntry entry = new AppCatalogEntry(); + while (appList.hasNext()) { + SolrDocument d = appList.next(); + entry.setOrg(d.get("org_s").toString()); + entry.setName(d.get("name_s").toString()); + entry.setDesc(d.get("desc_s").toString()); + yarnApp = mapper.readValue(d.get("yarnfile_s").toString(), + org.apache.hadoop.yarn.service.api.records.Service.class); + entry.setLike(Integer.parseInt(d.get("like_i").toString())); + entry.setDownload(Integer.parseInt(d.get("download_i").toString())); + download = entry.getDownload() + 1; + + // Update download count + docs.add(incrementDownload(d, download)); + } + + if (yarnApp != null) { + // Register deployed application instance with AppList + yarnApp.setName(name); + SolrInputDocument request = new SolrInputDocument(); + request.addField("type_s", "AppEntry"); + request.addField("id", name); + request.addField("name_s", name); + request.addField("app_s", entry.getOrg() + "/" + entry.getName()); + request.addField("yarnfile_s", mapper.writeValueAsString(yarnApp)); + docs.add(request); + } + + // Commit Solr changes. + UpdateResponse detailsResponse = solr.add(docs); + if (detailsResponse.getStatus() != 0) { + throw new IOException( + "Unable to register docker instance with application entry."); + } + solr.commit(); + return yarnApp; + } + + private SolrInputDocument incrementDownload(SolrDocument doc, long download) { + Collection names = doc.getFieldNames(); + SolrInputDocument s = new SolrInputDocument(); + for (String name : names) { + if (!name.equals("_version_")) { + s.addField(name, doc.getFieldValues(name)); + } + } + download++; + s.setField("download_i", download); + return s; + } + + public void deleteApp(String id) { + SolrClient solr = getSolrClient(); + try { + solr.deleteById(id); + solr.commit(); + } catch (SolrServerException | IOException e) { + LOG.error("Error in removing deployed application: " + id, e); + } + } + + public void register(Application app) throws IOException { + Collection docs = new HashSet(); + SolrClient solr = getSolrClient(); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + try { + SolrInputDocument buffer = new SolrInputDocument(); + buffer.setField("id", + java.util.UUID.randomUUID().toString().substring(0, 11)); + buffer.setField("org_s", app.getOrganization()); + buffer.setField("name_s", app.getName()); + buffer.setField("desc_s", app.getDescription()); + if (app.getIcon() != null) { + buffer.setField("icon_s", app.getIcon()); + } + buffer.setField("type_s", "AppCatalogEntry"); + buffer.setField("like_i", 0); + buffer.setField("download_i", 0); + + // Keep only YARN data model for yarnfile field + String yarnFile = mapper.writeValueAsString(app); + LOG.info("app:" + yarnFile); + org.apache.hadoop.yarn.service.api.records.Service yarnApp = mapper + .readValue(yarnFile, + org.apache.hadoop.yarn.service.api.records.Service.class); + buffer.setField("yarnfile_s", mapper.writeValueAsString(yarnApp)); + + docs.add(buffer); + // Commit Solr changes. + UpdateResponse detailsResponse = solr.add(docs); + if (detailsResponse.getStatus() != 0) { + throw new IOException( + "Unable to register application in Application Catalog."); + } + solr.commit(); + } catch (SolrServerException | IOException e) { + throw new IOException( + "Unable to register application in Application Catalog. " + + e.getMessage()); + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/package-info.java new file mode 100644 index 00000000000..d87ee1d18d5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/application/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Application Catalog Integration package. **/ +package org.apache.hadoop.yarn.appcatalog.application; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppCatalogController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppCatalogController.java new file mode 100644 index 00000000000..ce1d376d3db --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppCatalogController.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.controller; + +import java.io.IOException; +import java.util.List; + +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + +import org.apache.hadoop.yarn.appcatalog.application.YarnSolrClient; +import org.apache.hadoop.yarn.appcatalog.model.AppCatalogEntry; +import org.apache.hadoop.yarn.appcatalog.model.Application; + +/** + * Application catalog controller REST API use for searching, register and + * recommend application by YARN Application catalog UI. + * + */ +@Path("/ws/v2/app_catalog") +public class AppCatalogController { + + public AppCatalogController() { + } + + /** + * Display the most frequently used applications on appcatalog home page. + * + * @apiGroup AppCatalogController + * @apiName get + * @api {get} /ws/v2/app_catalog/recommended Display recommended + * applications. + * @apiSuccess {Object} AppEntry Application configuration. + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * [ + * { + * "id":"96b7833a-e3", + * "org":"Hortonworks", + * "name":"LAMP", + * "desc":"Linux Apache MySQL PHP web application", + * "icon":"/css/img/hwx_logo.png", + * "like":0, + * "download":0, + * "app":null + * }, + * { + * ... + * } + * ] + * @return - List of YARN applications + */ + @GET + @Path("recommended") + @Produces(MediaType.APPLICATION_JSON) + public List get() { + YarnSolrClient sc = new YarnSolrClient(); + return sc.getRecommendedApps(); + } + + /** + * Search for yarn applications from solr. + * + * @apiGroup AppCatalogController + * @apiName search + * @api {get} /ws/v2/app_catalog/search Find application from appcatalog. + * @apiParam {String} q Keyword to search. + * @apiSuccess {Object} AppCatalogEntry List of matched applications. + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * [ + * { + * "id":"96b7833a-e3", + * "org":"Hortonworks", + * "name":"LAMP", + * "desc":"Linux Apache MySQL PHP web application", + * "icon":"/css/img/hwx_logo.png", + * "like":0, + * "download":0, + * "app":null + * }, + * { + * ... + * } + * ] + * @param keyword - search for keyword + * @return - List of YARN applications matching keyword search. + */ + @GET + @Path("search") + @Produces(MediaType.APPLICATION_JSON) + public List search(@QueryParam("q") String keyword) { + YarnSolrClient sc = new YarnSolrClient(); + return sc.search(keyword); + } + + /** + * Register an application. + * + * @apiGroup AppCatalogController + * @apiName register + * @api {post} /ws/v2/app_catalog/register Register an application in + * appcatalog. + * @apiParam {Object} app Application definition. + * @apiParamExample {json} Request-Example: + * { + * "name": "Jenkins", + * "organization": "Jenkins-ci.org", + * "description": "The leading open source automation server", + * "icon": "/css/img/jenkins.png", + * "lifetime": "3600", + * "components": [ + * { + * "name": "jenkins", + * "number_of_containers": 1, + * "artifact": { + * "id": "eyang-1.openstacklocal:5000/jenkins:latest", + * "type": "DOCKER" + * }, + * "launch_command": "", + * "resource": { + * "cpus": 1, + * "memory": "2048" + * }, + * "configuration": { + * "env": { + * }, + * "files": [ + * ] + * } + * } + * ], + * "quicklinks": { + * "Jenkins": "http://jenkins.${SERVICE_NAME}.${USER}.${DOMAIN}:8080/" + * } + * } + * @apiSuccess {String} Response Application register result. + * @apiError BadRequest Error in process application registration. + * @param app - Yarnfile in JSON form + * @return Web response + */ + @POST + @Path("register") + @Produces(MediaType.APPLICATION_JSON) + public Response register(Application app) { + try { + if (app.getName()==null) { + throw new IOException("Application name can not be empty."); + } + if (app.getOrganization()==null) { + throw new IOException("Application organization can not be empty."); + } + if (app.getDescription()==null) { + throw new IOException("Application description can not be empty."); + } + YarnSolrClient sc = new YarnSolrClient(); + sc.register(app); + } catch (IOException e) { + return Response.status(Status.BAD_REQUEST).entity(e.getMessage()).build(); + } + return Response.status(Status.ACCEPTED).build(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppDetailsController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppDetailsController.java new file mode 100644 index 00000000000..b51be46b906 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppDetailsController.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.controller; + +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + +import org.apache.hadoop.yarn.appcatalog.application.YarnClient; +import org.apache.hadoop.yarn.appcatalog.application.YarnSolrClient; +import org.apache.hadoop.yarn.appcatalog.model.AppEntry; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ServiceState; + +import com.fasterxml.jackson.core.JsonProcessingException; + +/** + * REST API for query deployed application configuration and status. + * + */ +@Path("/ws/v2/app_details") +public class AppDetailsController { + + public AppDetailsController() { + } + + /** + * List detail information about the deployed application. + * + * @apiGroup AppDetailController + * @apiName getDetails + * @api {get} /ws/v2/app_details/config/{id} Check config of + * application instance. + * @apiParam {String} id Application ID to fetch configuration. + * @apiSuccess {Object} AppEntry Application configuration. + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "id":"howita_man", + * "name":"howita_man", + * "app":"Jenkins-ci.org/Jenkins", + * "yarnfile":{ + * "name":"howita_man", + * "lifetime":3600, + * "containers":[ + * ], + * "components":[ + * { + * "name":"jenkins", + * "dependencies":[ + * ], + * "artifact":{ + * "id":"eyang-1.openstacklocal:5000/jenkins:latest", + * "type":"DOCKER" + * }, + * "launch_command":"", + * "resource":{ + * "uri":null, + * "profile":null, + * "cpus":1, + * "memory":"2048" + * }, + * "number_of_containers":1, + * "run_privileged_container":false, + * "configuration":{ + * "properties":{ + * }, + * "env":{ + * }, + * "files":[ + * ] + * }, + * "quicklinks":[ + * ], + * "containers":[ + * ] + * } + * ], + * "configuration":{ + * "properties":{ + * }, + * "env":{ + * }, + * "files":[ + * ] + * }, + * "quicklinks":{ + * "Jenkins UI":"http://jenkins.howita_man.yarn.${DOMAIN}:8080/" + * } + * } + * } + * @param id - Application ID + * @return application entry- + */ + @Path("config/{id}") + @GET + @Produces(MediaType.APPLICATION_JSON) + public AppEntry getDetails(@PathParam("id") String id) { + YarnSolrClient sc = new YarnSolrClient(); + return sc.findAppEntry(id); + } + + /** + * Check application status. + * + * @apiGroup AppDetailController + * @apiName getStatus + * @api {get} /ws/v2/app_details/status/{id} Check status of + * application instance. + * @apiParam {String} id Application ID to check. + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * { + * "id":"howita_man", + * "name":"howita_man", + * "app":"Jenkins-ci.org/Jenkins", + * "yarnfile":{ + * "name":"howita_man", + * "lifetime":3099, + * "containers":[ + * ], + * "id":"application_1503694240849_0043", + * "components":[ + * { + * "name":"jenkins", + * "dependencies":[ + * ], + * "artifact":{ + * "id":"eyang-1.openstacklocal:5000/jenkins:latest", + * "type":"DOCKER" + * }, + * "launch_command":"", + * "resource":{ + * "uri":null, + * "profile":null, + * "cpus":1, + * "memory":"2048" + * }, + * "number_of_containers":1, + * "run_privileged_container":false, + * "configuration":{ + * "properties":{ + * }, + * "env":{ + * }, + * "files":[ + * ] + * }, + * "quicklinks":[ + * ], + * "containers":[ + * { + * "id":"container_1503694240849_0043_01_000002", + * "launch_time":1504630535403, + * "bare_host":"eyang-4.openstacklocal", + * "state":"READY", + * "component_name":"jenkins-0" + * } + * ] + * } + * ], + * "configuration":{ + * "properties":{ + * }, + * "env":{ + * }, + * "files":[ + * ] + * }, + * "quicklinks":{ + * "Jenkins UI":"http://jenkins.howita_man.yarn.${DOMAIN}:8080/" + * } + * } + * } + * @apiSuccess {Object} text Give status + * @param id - Application ID + * @return application entry + */ + @Path("status/{id}") + @GET + @Produces(MediaType.APPLICATION_JSON) + public AppEntry getStatus(@PathParam("id") String id) { + YarnSolrClient sc = new YarnSolrClient(); + AppEntry appEntry = sc.findAppEntry(id); + YarnClient.getStatus(appEntry); + return appEntry; + } + + /** + * Stop an application. + * + * @apiGroup AppDetailController + * @apiName stopApp + * @api {post} /ws/v2/app_details/stop/{id} Stop one instance of + * application. + * @apiParam {String} id Application ID to stop. + * @apiSuccess {String} text Give deployment status + * @apiError BadRequest Requested application does not stop. + * @param id - Application ID + * @return Web response code + */ + @Path("stop/{id}") + @POST + @Produces(MediaType.APPLICATION_JSON) + public Response stopApp(@PathParam("id") String id) { + YarnSolrClient sc = new YarnSolrClient(); + AppEntry app = sc.findAppEntry(id); + Service yarnApp = app.getYarnfile(); + yarnApp.setState(ServiceState.STOPPED); + try { + YarnClient.stopApp(yarnApp); + } catch (JsonProcessingException e) { + return Response.status(Status.BAD_REQUEST).build(); + } + return Response.ok().build(); + } + + /** + * Restart an application. + * + * @apiGroup AppDetailController + * @apiName restartApp + * @api {post} /ws/v2/app_details/restart/{id} Restart one instance of + * application. + * @apiParam {String} id Application ID to restart. + * @apiSuccess {String} text Give deployment status + * @apiError BadRequest Requested application does not restart. + * @param id - Application ID + * @return Web response code + */ + @Path("restart/{id}") + @POST + @Produces(MediaType.APPLICATION_JSON) + public Response restartApp(@PathParam("id") String id) { + YarnSolrClient sc = new YarnSolrClient(); + AppEntry app = sc.findAppEntry(id); + Service yarnApp = app.getYarnfile(); + yarnApp.setState(ServiceState.STARTED); + try { + YarnClient.restartApp(yarnApp); + } catch (JsonProcessingException e) { + return Response.status(Status.BAD_REQUEST).build(); + } + return Response.ok().build(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppListController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppListController.java new file mode 100644 index 00000000000..855e59e2ba1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/AppListController.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.controller; + +import java.io.IOException; +import java.util.List; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + +import org.apache.hadoop.yarn.appcatalog.application.YarnClient; +import org.apache.hadoop.yarn.appcatalog.application.YarnSolrClient; +import org.apache.hadoop.yarn.appcatalog.model.AppEntry; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.solr.client.solrj.SolrServerException; + +/** + * Application Catalog REST API used for display deployed application, + * trigger application to deploy, or shutdown deployed application. + * + */ +@Path("/ws/v2/app_list") +@Produces({ MediaType.APPLICATION_JSON }) +public class AppListController { + + public AppListController() { + } + + /** + * Get Application List. + * + * @apiGroup AppListController + * @apiName get + * @api {get} /ws/v2/app_list Get list of deployed applications. + * @apiSuccess {Object[]} List<AppEntry> List of deployed + * Applications. + * @apiSuccessExample {json} Success-Response: + * HTTP/1.1 200 OK + * [ + * { + * "id":"howita-man", + * "name":"howita-man", + * "app":"Jenkins-ci.org/Jenkins", + * "yarnfile":{ + * "name":"howita_man", + * "lifetime":3600, + * "containers":[ + * ], + * "components":[ + * { + * "name":"jenkins", + * "dependencies":[ + * ], + * "artifact":{ + * "id":"eyang-1.openstacklocal:5000/jenkins:latest", + * "type":"DOCKER" + * }, + * "launch_command":"", + * "resource":{ + * "uri":null, + * "profile":null, + * "cpus":1, + * "memory":"2048" + * }, + * "number_of_containers":1, + * "run_privileged_container":false, + * "configuration":{ + * "properties":{ + * }, + * "env":{ + * }, + * "files":[ + * ] + * }, + * "quicklinks":[ + * ], + * "containers":[ + * ] + * } + * ], + * "configuration":{ + * "properties":{ + * }, + * "env":{ + * }, + * "files":[ + * ] + * }, + * "quicklinks":{ + * "Jenkins":"http://jenkins.${SERVICE_NAME}.${USER}.${DOMAIN}:8080" + * } + * } + * }, + * { + * ... + * } + * ] + * @return - Active application deployed by current user. + */ + @GET + @Produces(MediaType.APPLICATION_JSON) + public List getList() { + YarnSolrClient sc = new YarnSolrClient(); + return sc.listAppEntries(); + } + + /** + * Delete an application. + * + * @apiGroup AppListController + * @apiName delete + * @api {delete} /ws/v2/app_list Delete one instance of application. + * @apiParam {String} id Application name to delete. + * @apiSuccess {String} text Delete request accepted + * @param id - application ID + * @param name - application name + * @return Web response + */ + @DELETE + @Path("{id}/{name}") + @Produces(MediaType.APPLICATION_JSON) + public Response delete(@PathParam("id") String id, + @PathParam("name") String name) { + YarnSolrClient sc = new YarnSolrClient(); + sc.deleteApp(id); + YarnClient.deleteApp(name); + return Response.status(Status.ACCEPTED).build(); + } + + /** + * Deploy an application. + * + * @apiGroup AppListController + * @apiName deploy + * @api {post} /ws/v2/app_list/{id} Deploy one instance of application. + * @apiParam {String} id Application ID to deploy. + * @apiSuccess {String} text Give deployment status + * @apiError BadRequest Unable to deploy requested application. + * @param id - application ID + * @return Web response + */ + @POST + @Path("{id}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public Response deploy(@PathParam("id") String id) { + YarnSolrClient sc = new YarnSolrClient(); + Service app; + try { + app = sc.deployApp(id); + } catch (SolrServerException | IOException e) { + return Response.status(Status.BAD_REQUEST).entity(e.toString()).build(); + } + YarnClient.createApp(app); + String output = "{\"status\":\"Application deployed.\",\"id\":\"" + + app.getName() + "\"}"; + return Response.status(Status.ACCEPTED).entity(output).build(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/package-info.java new file mode 100644 index 00000000000..b182450c90f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/controller/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Application Catalog REST API package. **/ +package org.apache.hadoop.yarn.appcatalog.controller; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppCatalogEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppCatalogEntry.java new file mode 100644 index 00000000000..5581094bb71 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppCatalogEntry.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Data model for registering an application in application catalog. + * + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.PUBLIC_MEMBER) +public class AppCatalogEntry { + private String id; + private String org; + private String name; + private String desc; + private String icon = "css/img/feather.png"; + private long like; + private long download; + private Application app; + + public String getOrg() { + return org; + } + + public void setOrg(String org) { + this.org = org; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public long getLike() { + return like; + } + + public void setLike(long like) { + this.like = like; + } + + public long getDownload() { + return download; + } + + public void setDownload(long download) { + this.download = download; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Application getApp() { + return app; + } + + public void setApp(Application app) { + this.app = app; + } + + public String getIcon() { + return icon; + } + + public void setIcon(String icon) { + this.icon = icon; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java new file mode 100644 index 00000000000..5299d54a80c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.model; + +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Data model used to describe configuration of a deployed application. + * + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.PUBLIC_MEMBER) +public class AppDetails { + private String image; + private String version; + private List ports; + private List volumes; + private List env; + + public String getImage() { + return image; + } + + public void setImage(String image) { + this.image = image; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public List getPorts() { + return ports; + } + + public void setPorts(List ports2) { + this.ports = ports2; + } + + public List getVolumes() { + return volumes; + } + + public void setVolumes(List volumes) { + this.volumes = volumes; + } + + public List getEnv() { + return env; + } + + public void setEnv(List env) { + this.env = env; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppEntry.java new file mode 100644 index 00000000000..61a704a22d2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppEntry.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlType; + +/** + * Data model used to describe an deployed application. + * + */ +@XmlType(namespace="http://hadoop.apache.org", name="AppEntry") +@XmlRootElement +@XmlAccessorType(XmlAccessType.PUBLIC_MEMBER) +public class AppEntry { + private String id; + private String name; + private String app; + private org.apache.hadoop.yarn.service.api.records.Service yarnfile; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getApp() { + return app; + } + + public void setApp(String app) { + this.app = app; + } + + public org.apache.hadoop.yarn.service.api.records.Service getYarnfile() { + return yarnfile; + } + + public void setYarnfile( + org.apache.hadoop.yarn.service.api.records.Service yarnfile) { + this.yarnfile = yarnfile; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java new file mode 100644 index 00000000000..5a92ab356cf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.model; + +import javax.xml.bind.annotation.XmlRootElement; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +/** + * Data model used to describe an YARN application. + * + */ +@XmlRootElement +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonPropertyOrder({ "organization", "name", "description", "icon" }) +public class Application extends + org.apache.hadoop.yarn.service.api.records.Service { + private static final long serialVersionUID = -1776203219305414248L; + + private String organization; + private String description; + private String icon; + + @JsonProperty("organization") + public String getOrganization() { + return organization; + } + public void setOrganization(String organization) { + this.organization = organization; + } + + @JsonProperty("description") + public String getDescription() { + return description; + } + public void setDescription(String description) { + this.description = description; + } + + @JsonProperty("icon") + public String getIcon() { + return icon; + } + public void setIcon(String icon) { + this.icon = icon; + } + + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/package-info.java new file mode 100644 index 00000000000..1816b4285c9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/model/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Application Catalog Data Model package. **/ +package org.apache.hadoop.yarn.appcatalog.model; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/utils/RandomWord.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/utils/RandomWord.java new file mode 100644 index 00000000000..0f0cb99b1cf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/utils/RandomWord.java @@ -0,0 +1,427 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.appcatalog.utils; + +import java.util.Arrays; +import java.util.Random; + +/** + * Utility class to randomly generate English sound alike word. + * + */ +public class RandomWord { + + protected RandomWord() { + } + + public static String getNewWord(int wordLength) throws WordLengthException { + if (wordLength < 3 || wordLength > 15) { + throw new WordLengthException("Word length error, words must be between" + + " 3 and 15 characters long."); + } + String theNewWord = "e"; + while (theNewWord.equals("e")) { + try { + theNewWord = generateRandomWord(wordLength); + } catch (Exception e) { + } + } + return theNewWord; + } + + private static String generateRandomWord(int wordLength) { + String randomWord; + // --------------------------------------------------------------------- + // Bigram source and general concept based on + // https://github.com/scrollback/scrollback & described in + // https://www.hackerearth.com/notes/random-pronouncable-text-generator/ + String[] startBiGram = {"TH", "OF", "AN", "IN", "TO", "CO", "BE", "HE", + "RE", "HA", "WA", "FO", "WH", "MA", "WI", "ON", "HI", "PR", "ST", + "NO", "IS", "IT", "SE", "WE", "AS", "CA", "DE", "SO", "MO", "SH", + "DI", "AL", "AR", "LI", "WO", "FR", "PA", "ME", "AT", "SU", "BU", + "SA", "FI", "NE", "CH", "PO", "HO", "DO", "OR", "UN", "LO", "EX", + "BY", "FA", "LA", "LE", "PE", "MI", "SI", "YO", "TR", "BA", "GO", + "BO", "GR", "TE", "EN", "OU", "RA", "AC", "FE", "PL", "CL", "SP", + "BR", "EV", "TA", "DA", "AB", "TI", "RO", "MU", "EA", "NA", "SC", + "AD", "GE", "YE", "AF", "AG", "UP", "AP", "DR", "US", "PU", "CE", + "IF", "RI", "VI", "IM", "AM", "KN", "OP", "CR", "OT", "JU", "QU", + "TW", "GA", "VA", "VE", "PI", "GI", "BI", "FL", "BL", "EL", "JO", + "FU", "HU", "CU", "RU", "OV", "MY", "OB", "KE", "EF", "PH", "CI", + "KI", "NI", "SL", "EM", "SM", "VO", "MR", "WR", "ES", "DU", "TU", + "AU", "NU", "GU", "OW", "SY", "JA", "OC", "EC", "ED", "ID", "JE", + "AI", "EI", "SK", "OL", "GL", "EQ", "LU", "AV", "SW", "AW", "EY", + "TY"}; + String[] lookupBiGram = {"TH", "AN", "IN", "IO", "EN", "TI", "FO", + "HE", "HA", "HI", "TE", "AT", "ER", "AL", "WA", "VE", "CO", "RE", + "IT", "WI", "ME", "NC", "ON", "PR", "AR", "ES", "EV", "ST", "EA", + "IV", "EC", "NO", "OU", "PE", "IL", "IS", "MA", "AV", "OM", "IC", + "GH", "DE", "AI", "CT", "IG", "ID", " OR", "OV", "UL", "YO", "BU", + "RA", "FR", "RO", "WH", "OT", "BL", "NT", "UN", "TR", "HO", "AC", + "TU", "WE", "CA", "SH", "UR", "IE", "PA", "TO", "EE", "LI", "RI", + "UG", "AM", "ND", "US", "LL", "AS", "TA", "LE", "MO", "WO", "MI", + "AB", "EL", "IA", "NA", "SS", "AG", "TT", "NE", "PL", " LA", "OS", + "CE", "DI", "BE", "AP", "SI", "NI", "OW", "SO", "AK", "CH", "EM", + "IM", "SE", "NS", "PO", "EI", "EX", "KI", "UC", "AD", "GR", "IR", + "NG", "OP", "SP", "OL", "DA", "NL", "TL", "LO", "BO", "RS", "FE", + "FI", "SU", "GE", "MP", "UA", "OO", "RT", "SA", "CR", "FF", "IK", + "MB", " KE", "FA", "CI", "EQ", "AF", "ET", "AY", "MU", "UE", "HR", + "TW", "GI", "OI", "VI", "CU", "FU", "ED", "QU", "UT", "RC", "OF", + "CL", "FT", "IZ", "PP", "RG", "DU", "RM", "YE", "RL", "DO", "AU", + "EP", "BA", "JU", "RD", "RU", "OG", "BR", "EF", "KN", "LS", "GA", + "PI", "YI", "BI", "IB", "UB", "VA", "OC", "IF", "RN", "RR", "SC", + "TC", "CK", "DG", "DR", "MM", "NN", "OD", "RV", "UD", "XP", "JE", + "UM", "EG", "DL", "PH", "SL", "GO", "CC", "LU", "OA", "PU", "UI", + "YS", "ZA", "HU", "MR", "OE", "SY", "EO", "TY", "UP", "FL", "LM", + "NF", "RP", "OH", "NU", "XA", "OB", "VO", "DM", "GN", " LD", "PT", + "SK", "WR", "JO", "LT", "YT", "UF", "BJ", "DD", "EY", "GG", "GL", + "GU", "HT", "LV", "MS", "NM", "NV", "OK", "PM", "RK", "SW", "TM", + "XC", "ZE", "AW", "SM"}; + String[][][] nextCharLookup = { + {{"E", "A", "I", "O", "R"}, {"E", "O"}}, + {{"D", "T", "Y", "C", "S", "G", "N", "I", "O", "E", "A", "K"}, + {"D", "T", "Y", "S", "G", "O", "E", "A", "K"}}, + {{"G", "T", "E", "D", "S", "C", "A", "I", "K", "V", "U", "N", "F"}, + {"G", "T", "E", "D", "S", "A", "K"}}, + {{"N", "U", "R"}, {"N", "U", "R"}}, + {{"T", "C", "D", "S", "E", "I", "G", "O", "N", "A"}, + {"T", "D", "S", "E", "G", "O", "A"}}, + {{"O", "N", "C", "V", "M", "L", "E", "T", "S", "A", "R", "F"}, + {"N", "C", "M", "L", "E", "T", "S", "A", "R", "F"}}, + {{"R", "U", "O", "L"}, {"R", "U", "O", "L"}}, + {{"R", "N", "Y", "S", "M", "I", "A", "L", "D", "T"}, + {"R", "N", "Y", "S", "M", "A", "L", "D", "T"}}, + {{"T", "D", "V", "N", "S", "R", "P", "L"}, + {"T", "D", "N", "S", "R", "L"}}, + {{"S", "N", "C", "M", "L", "P", "G", "T", "R", "E"}, + {"S", "N", "C", "M", "L", "P", "G", "T", "R", "E"}}, + {{"R", "D", "N", "S", "M", "L", "E", "C", "A"}, + {"R", "D", "N", "S", "M", "L", "E", "A"}}, + {{"I", "E", "T", "H", "U", "O", "C"}, {"E", "H", "O"}}, + {{"E", "S", "I", "A", "N", "Y", "T", "V", "M", "R", "O", "L", "G", + "F", "C"}, {"E", "S", "A", "N", "Y", "T", "M"}}, + {{"L", "S", "I", "T", "E", "U", "O", "M", "K", "F", "A"}, + {"L", "S", "T", "E", "F"}}, + {{"S", "Y", "R", "T", "N", "L"}, {"S", "Y", "R", "T", "N", "L"}}, + {{"R", "N", "L", "S", "D"}, {"R", "N", "L", "S", "D"}}, + {{"N", "M", "U", "R", "L", "V", "S", "O"}, + {"N", "M", "U", "R", "L", "O"}}, + {{"S", "A", "D", "N", "E", "C", "L", "T", "P", "M", "V", "G", "F", + "Q"}, {"S", "A", "D", "N", "E", "L", "T", "P", "M"}}, + {{"H", "I", "Y", "E", "S", "T", "A", "U"}, + {"H", "Y", "E", "S", "A"}}, + {{"T", "L", "N", "S"}, {"T", "L", "N", "S"}}, + {{"N", "R", "D", "T", "S", "M", "A"}, + {"N", "R", "D", "T", "S", "M", "A"}}, + {{"E", "I", "H", "T", "R", "O", "L"}, {"E", "H", "T"}}, + {{"S", "E", "T", "G", "A", "D", "L", "C", "V", "O", "I", "F"}, + {"S", "E", "T", "G", "A", "D", "O"}}, + {{"O", "E", "I", "A"}, {"E", "A"}}, + {{"E", "T", "D", "Y", "S", "I", "R", "L", "M", "K", "G", "A", "O", + "N", "C"}, {"E", "T", "D", "Y", "S", "M", "K", "A", "N"}}, + {{"S", "T", "E", "I", "P", "U", "C"}, {"S", "T", "E"}}, + {{"E", "I"}, {"E"}}, + {{"A", "R", "I", "E", "O", "U", "S"}, {"A", "E", "O", "S"}}, + {{"R", "S", "T", "D", "L", "C", "N", "V", "M", "K"}, + {"R", "S", "T", "D", "L", "N", "M"}}, + {{"E", "I", "A"}, {"E"}}, + {{"T", "O", "I", "E", "A", "U", "R", "H"}, {"T", "E", "H"}}, + {{"T", "W", "R", "U", "N", "M"}, {"T", "W", "R", "U", "N", "M"}}, + {{"T", "L", "R", "N", "S", "G", "P", "B"}, + {"T", "L", "R", "N", "S", "P"}}, + {{"R", "N", "C", "A", "D", "T", "O"}, {"R", "N", "A", "D", "T"}}, + {{"L", "E", "I", "Y", "D", "A"}, {"L", "E", "Y", "D"}}, + {{"T", "H", "S", "I", "E", "C", "M"}, {"T", "H", "S", "E", "M"}}, + {{"N", "T", "L", "K", "D", "S", "I", "G"}, + {"N", "T", "L", "D", "S"}}, + {{"E", "I", "A"}, {"E"}}, + {{"E", "P", "M", "I", "A"}, {"E"}}, + {{"A", "H", "E", "I", "T", "K", "U", "S"}, + {"H", "E", "T", "K", "S"}}, + {{"T"}, {"T"}}, + {{"R", "N", "S", "D", "A", "V", "P", "T", "M", "L", "F"}, + {"R", "N", "S", "D", "A", "P", "T", "M", "L"}}, + {{"N", "D", "R", "L", "T"}, {"N", "D", "R", "L", "T"}}, + {{"I", "E", "U", "S", "O"}, {"E", "S", "O"}}, + {{"H", "N", "I"}, {"H", "N"}}, {{"E"}, {"E"}}, + {{"E", "T", "M", "D", "S", "K", "I", "Y", "L", "G", "A", "R", "N", + "C"}, {"E", "T", "M", "D", "S", "K", "Y", "A", "N"}}, + {{"E", "I"}, {"E"}}, + {{"D", "T", "A", "L"}, {"D", "T", "L"}}, {{"U"}, {"U"}}, + {{"T", "S", "R", "I"}, {"T", "S", "R"}}, + {{"T", "N", "L", "C", "I", "M", "D", "S", "R", "P", "G", "B"}, + {"T", "N", "L", "M", "D", "S", "R"}}, + {{"O", "E", "A"}, {"E", "A"}}, + {{"M", "U", "V", "P", "N", "W", "S", "O", "L", "D", "C", "B", "A", + "T", "G"}, {"M", "U", "P", "N", "W", "O", "L", "D", "T"}}, + {{"I", "E", "O", "A"}, {"E", "O"}}, + {{"H", "E", "T", "I"}, {"H", "E"}}, + {{"E", "I", "Y", "O", "A"}, {"E", "Y"}}, + {{"E", "I", "S", "R", "O", "A", "L", "Y", "U", "H"}, + {"E", "S", "O", "A", "Y", "H"}}, + {{"D", "T", "I", "C", "G"}, {"D", "T", "G"}}, + {{"A", "I", "O", "E", "U", "Y"}, {"A", "E", "Y"}}, + {{"U", "W", "S", "R", "L", "O", "M", "T", "P", "N", "D"}, + {"U", "W", "R", "L", "O", "M", "T", "P", "N", "D"}}, + {{"T", "E", "K", "H", "C", "R", "I"}, {"T", "E", "K", "H"}}, + {{"R", "D", "A", "T"}, {"R", "T"}}, + {{"R", "L", "E", "V", "S", "N", "A"}, + {"R", "L", "E", "S", "N", "A"}}, + {{"L", "N", "T", "R", "U", "S", "M", "P"}, + {"L", "N", "T", "R", "S", "M"}}, + {{"E", "O", "I", "A"}, {"E", "O"}}, + {{"E", "N", "T", "S", "I", "A", "Y", "R", "P", "C"}, + {"E", "N", "T", "S", "A", "Y"}}, + {{"S", "N", "D", "T", "W", "V", "R", "L", "F"}, + {"S", "N", "D", "T", "W", "R", "L"}}, + {{"R", "T", "S", "N", "L", "I", "C"}, {"R", "T", "S", "N", "L"}}, + {{"R", "O", "N", "W", "P", "M", "L"}, + {"R", "O", "N", "W", "P", "M", "L"}}, + {{"N", "D", "T", "M", "S", "R", "P", "L", "K"}, + {"N", "D", "T", "M", "S", "R", "P", "L", "K"}}, + {{"N", "T", "S", "C", "K", "G", "E", "F", "Z", "V", "O", "M", "A"}, + {"N", "T", "S", "C", "G", "E", "F", "M", "A"}}, + {{"N", "E", "C", "T", "S", "G", "A", "V", "O", "P", "M", "L", "D", + "B"}, {"N", "E", "C", "T", "S", "G", "A", "P", "M", "L", "D"}}, + {{"H", "G"}, {"H"}}, {{"E", "P", "I", "O", "A"}, {"E"}}, + {{"E", "I", "S", "A", "U", "O"}, {"E", "S", "O"}}, + {{"E", "T", "I", "S", "L", "H"}, {"E", "T", "S", "H"}}, + {{"Y", "E", "O", "I", "S", "A"}, {"Y", "E", "S"}}, + {{"T", "S", "E", "I", "U", "O", "K", "H"}, + {"T", "S", "E", "O", "H"}}, + {{"T", "N", "L", "I", "R", "K", "B", "G", "C"}, + {"T", "N", "L", "R"}}, + {{"S", "D", "A", "T", "C", "R", "N", "M", "G", "V", "F"}, + {"S", "D", "A", "T", "R", "N", "M"}}, + {{"R", "S", "V", "T", "U", "D"}, {"R", "T", "U", "D"}}, + {{"R", "U"}, {"R", "U"}}, + {{"N", "L", "S", "T", "C", "G"}, {"N", "L", "S", "T", "C", "G"}}, + {{"L", "O", "I"}, {} }, + {{"L", "Y", "I", "E", "F", "O", "A", "T", "S", "P", "D"}, + {"L", "Y", "E", "F", "T", "S", "D"}}, + {{"L", "N", "T"}, {"L", "N", "T"}}, + {{"L", "T", "R", "N", "M"}, {"L", "T", "R", "N", "M"}}, + {{"I", "E", "U", "O", "A"}, {"E", "O"}}, + {{"E", "A", "O"}, {"E", "O"}}, {{"E", "L", "I"}, {"E"}}, + {{"D", "S", "W", "R", "E", "Y", "V", "T", "L", "C", "A"}, + {"D", "S", "W", "R", "E", "Y", "T", "L", "A"}}, + {{"A", "E", "I", "Y", "O"}, {"E", "Y"}}, + {{"T", "N", "R", "S", "C", "Y", "W", "I", "B"}, + {"T", "N", "R", "S", "Y", "W"}}, + {{"T", "E", "S", "I"}, {"T", "E", "S"}}, + {{"S", "N", "R", "D", "P", "L", "I"}, + {"S", "N", "R", "D", "P", "L"}}, + {{"S", "N", "T", "D", "F", "E", "C", "A", "V", "R"}, + {"S", "N", "T", "D", "F", "E", "C", "A", "R"}}, + {{"R", "E", "C", "T", "L", "F", "S", "I", "G", "D", "A"}, + {"R", "E", "T", "L", "S", "D", "A"}}, + {{"P", "E", "A"}, {"E"}}, + {{"O", "N", "D", "T", "S", "G", "C", "B", "V", "M", "A"}, + {"N", "D", "T", "S", "G", "C", "M", "A"}}, + {{"N", "T", "S", "C", "Z", "O", "G", "F"}, + {"N", "T", "S", "C", "G", "F"}}, + {{"N", "E", "S", "I", "A"}, {"N", "E", "S"}}, + {{"N", "M", "U", "L", "C", "R"}, {"N", "M", "U", "L", "R"}}, + {{"E", "I"}, {"E"}}, + {{"E", "A", "I", "O", "U", "R"}, {"E", "O"}}, + {{"E", "S", "P", "O", "B", "A", "I"}, {"E", "S"}}, + {{"E", "P", "I", "A", "S", "M"}, {"E", "S"}}, + {{"D", "N", "L", "S", "R", "E", "C", "T", "V", "A"}, + {"D", "N", "L", "S", "R", "E", "T", "A"}}, + {{"T", "I", "E"}, {"T", "E"}}, + {{"S", "R", "N", "L", "W", "T", "I"}, {"R", "N", "L", "W", "T"}}, + {{"R", "N", "G", "T"}, {"R", "N", "G", "T"}}, + {{"P", "T", "I", "C", "A"}, {"T"}}, {{"N"}, {"N"}}, + {{"H", "T", "K", "E"}, {"H", "T", "K", "E"}}, + {{"E", "I", "Y", "V", "M", "D"}, {"E", "Y"}}, + {{"E", "A", "O"}, {"E", "A"}}, + {{"E", "S", "T", "L", "I"}, {"E", "S", "T"}}, + {{"E", "S", "L", "T", "R", "I"}, {"E", "S"}}, + {{"E", "P", "L"}, {"E"}}, {{"E", "O", "I", "A"}, {"E"}}, + {{"D", "L", "I", "O", "E", "U"}, {"D", "L", "E"}}, + {{"Y", "T", "R", "N"}, {"Y", "T", "R", "N"}}, + {{"Y"}, {"Y"}}, {{"Y", "E"}, {"Y", "E"}}, + {{"W", "N", "O", "S", "C", "V", "U", "T", "R", "P", "G"}, + {"W", "N", "O", "U", "T", "R", "P"}}, + {{"U", "T", "R", "O", "D", "A"}, {"U", "T", "R", "O", "D"}}, + {{"T", "E", "O", "I"}, {"T", "E", "O"}}, + {{"R", "E", "W", "L", "C", "A"}, {"R", "E", "W", "L", "A"}}, + {{"R", "N", "C", "E", "L", "G"}, {"R", "N", "C", "E", "L", "G"}}, + {{"R", "C", "P", "B", "M", "L", "A"}, {"R", "P", "M", "L"}}, + {{"N", "T", "S", "R", "D"}, {"N", "T", "S", "R", "D"}}, + {{"L", "O", "A", "T", "R", "E"}, {"T", "E"}}, + {{"L", "T", "R"}, {"L", "T", "R"}}, + {{"K", "D", "L", "T", "R", "N", "M"}, + {"K", "D", "L", "T", "R", "N", "M"}}, + {{"I", "H", "A", "E", "Y", "U", "S"}, {"H", "A", "E", "Y", "S"}}, + {{"I", "M", "Y", "N", "L"}, {"M", "Y", "N", "L"}}, + {{"E", "I", "O", "A"}, {"E", "A"}}, {{"E", "I"}, {"E"}}, + {{"E"}, {"E"}}, {{"E"}, {"E"}}, + {{"D", "N", "T", "S", "R", "E"}, {"D", "N", "T", "S", "R", "E"}}, + {{"C", "R", "M", "I"}, {"R", "M"}}, + {{"A", "T", "E", "S", "P", "N"}, {"A", "T", "E", "S", "P", "N"}}, + {{"U"}, {} }, {{"T", "F"}, {"T", "F"}}, + {{"T", "I", "H", "E", "Y", "W", "S", "A"}, + {"H", "E", "Y", "S", "A"}}, + {{"S", "E"}, {"S"}}, + {{"S", "N", "L", "C"}, {"S", "N", "L"}}, + {{"S", "N"}, {"S", "N"}}, {{"O", "E", "I"}, {"E"}}, + {{"O", "E"}, {"O", "E"}}, + {{"N", "V", "O", "C"}, {"N", "C"}}, {{"N"}, {"N"}}, + {{"N", "D", "S", "C", "T", "O", "L", "E"}, + {"N", "D", "S", "C", "T", "L", "E"}}, + {{"L", "R", "T", "S"}, {"L", "R", "T", "S"}}, + {{"L", "R", "N"}, {"L", "R", "N"}}, + {{"I", "U", "E"}, {"E"}}, {{"I", "E", "A"}, {"E"}}, + {{"I", "H", "E"}, {"H", "E"}}, {{"H", "E"}, {"H", "E"}}, + {{"F", "T"}, {"F", "T"}}, {{"E", "A", "U", "O"}, {"E"}}, + {{"E"}, {"E"}}, {{"E", "A"}, {"E"}}, + {{"E", "O", "R", "L"}, {"E"}}, {{"E", "A"}, {"E"}}, + {{"C", "S", "R", "A"}, {"S", "R"}}, + {{"A", "S", "I", "E"}, {"S", "E"}}, + {{"A", "S", "D"}, {"A", "S", "D"}}, + {{"Y", "D"}, {"Y", "D"}}, + {{"W", "N", "M", "E"}, {"W", "N", "M"}}, + {{"T", "S"}, {"T", "S"}}, + {{"T", "O", "E", "A"}, {"T", "E"}}, + {{"S", "C", "R", "N", "L"}, {"S", "R", "N", "L"}}, + {{"S"}, {"S"}}, {{"S", "E", "I"}, {"S", "E"}}, + {{"S", "N", "C"}, {"S", "N"}}, {{"R", "I"}, {} }, + {{"O", "I", "E", "A"}, {"E", "A"}}, + {{"O", "F", "U", "T", "E"}, {"F", "T", "E"}}, + {{"O", "E"}, {"O", "E"}}, {{"O"}, {"O"}}, + {{"N", "I", "T", "R"}, {"N", "T", "R"}}, + {{"N", "T", "R", "E", "C"}, {"N", "T", "R", "E", "C"}}, + {{"N"}, {"N"}}, {{"L", "T", "N"}, {"L", "T", "N"}}, + {{"L", "I", "E"}, {"E"}}, {{"L"}, {} }, + {{"L", "T", "R", "N"}, {"L", "T", "R", "N"}}, + {{"K", "I", "E", "C", "A"}, {"K", "E"}}, + {{"I", "F", "E", "T"}, {"F", "E", "T"}}, + {{"I", "E", "M", "A"}, {"E", "A"}}, + {{"I", "E", "Y", "O"}, {"E", "Y"}}, + {{"H", "R", "O", "I", "A"}, {"H"}}, {{"H"}, {"H"}}, + {{"E"}, {"E"}}, {{"E"}, {"E"}}, + {{"E", "O", "I", "A"}, {"E", "A"}}, + {{"E", "U", "I"}, {"E"}}, {{"E", "O", "I"}, {"E", "O"}}, + {{"E", "Y", "U"}, {"E", "Y"}}, {{"E", "I"}, {"E"}}, + {{"E", "I"}, {"E"}}, {{"E"}, {"E"}}, {{"C"}, {} }, + {{"B", "E"}, {"E"}}, {{"A", "R", "I", "E"}, {"E"}}, + {{"Y", "E"}, {"Y", "E"}}, + {{"Y", "O", "I", "E"}, {"Y", "O", "E"}}, + {{"Y", "A"}, {"Y"}}, {{"V", "T", "O"}, {"T", "O"}}, + {{"U", "O", "E"}, {"E"}}, + {{"T", "S", "M", "E", "D"}, {"T", "S", "M", "E"}}, + {{"T", "R", "D"}, {"T", "R", "D"}}, + {{"T", "R", "L", "B"}, {"T", "R", "L"}}, + {{"T", "R", "L"}, {"T", "R", "L"}}, {{"T"}, {"T"}}, + {{"T"}, {"T"}}, + {{"S", "R", "N", "M"}, {"S", "R", "N", "M"}}, + {{"S"}, {"S"}}, {{"S"}, {"S"}}, {{"S"}, {"S"}}, + {{"R", "P"}, {"R", "P"}}, {{"P"}, {} }, {{"P", "O"}, {} }, + {{"O", "E"}, {"E"}}, {{"O"}, {} }, {{"O"}, {} }, + {{"O"}, {} }, {{"N"}, {} }, {{"M"}, {"M"}}, + {{"M"}, {"M"}}, {{"L"}, {} }, {{"L"}, {"L"}}, + {{"I"}, {} }, {{"I"}, {} }, {{"I", "E"}, {"E"}}, + {{"I"}, {} }, {{"I", "E"}, {"E"}}, {{"I"}, {} }, + {{"H"}, {} }, {{"H", "E"}, {"H", "E"}}, {{"H"}, {"H"}}, + {{"F"}, {"F"}}, {{"E"}, {} }, {{"E"}, {"E"}}, + {{"E"}, {} }, {{"E"}, {"E"}}, {{"E", "A"}, {"E"}}, + {{"E"}, {"E"}}, {{"E"}, {"E"}}, {{"E"}, {"E"}}, + {{"E"}, {"E"}}, {{"E"}, {"E"}}, {{"E"}, {"E"}}, + {{"E"}, {"E"}}, {{"E"}, {"E"}}, {{"E"}, {"E"}}, + {{"E"}, {"E"}}, {{"E"}, {"E"}}, {{"E"}, {"E"}}, + {{"D"}, {"D"}}, {{"A"}, {} }, {{"A"}, {} } }; + // ------------------------------------------------------------------------ + randomWord = startBiGram[indexGenerator(startBiGram.length)]; + int flag = 0; + int count = 0; + String previousWord; + while (randomWord.length() != wordLength) { + previousWord = randomWord; + randomWord = addCharacter(startBiGram, wordLength, randomWord, + lookupBiGram, nextCharLookup, flag); + if (previousWord.equals(randomWord)) { + count++; + } else { + flag = 0; + } + if (count == 5) { + flag = 1; + count++; + } else if (count == 20) { + randomWord = startBiGram[indexGenerator(startBiGram.length)]; + count = 0; + } + } + return randomWord; + } + + private static String addCharacter(String[] startBiGram, int desiredLength, + String currentWord, String[] lookupBiGram, String[][][] nextCharLookup, + int flag) { + int mainIndex = getLookupIndex(currentWord, lookupBiGram); + int type = 0; + if (currentWord.length() == (desiredLength - 1)) { + type = 1; + } + while (mainIndex < 0 || mainIndex > 263 + || nextCharLookup[mainIndex][type].length <= 0) { + if (currentWord.length() == 2) { + return startBiGram[indexGenerator(startBiGram.length)]; + } + if (flag == 1) { + currentWord = backtrack(currentWord, 2); + flag = 0; + } else { + currentWord = backtrack(currentWord, 1); + } + mainIndex = getLookupIndex(currentWord, lookupBiGram); + if (type == 1) { + type = 0; + } + } + String updatedWord = currentWord + + getNextCharacter(type, mainIndex, nextCharLookup); + return updatedWord; + } + + private static int indexGenerator(int arrayLength) { + int theIndex; + Random generator = new Random(); + theIndex = generator.nextInt(arrayLength); + return theIndex; + } + + private static String getNextCharacter(int type, int mainIndex, + String[][][] theCharacterVault) { + String nextChar; + int i = indexGenerator(theCharacterVault[mainIndex][type].length); + nextChar = theCharacterVault[mainIndex][type][i]; + return nextChar; + } + + private static String backtrack(String theWord, int numberChars) { + theWord = theWord.substring(0, theWord.length() - numberChars); + return theWord; + } + + private static int getLookupIndex(String theWord, String[] lookupArray) { + String lookupCharacters = theWord.substring(theWord.length() - 2); + int lookupIndex = Arrays.asList(lookupArray).indexOf(lookupCharacters); + return lookupIndex; + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/utils/WordLengthException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/utils/WordLengthException.java new file mode 100644 index 00000000000..b3d7c66b4b9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/utils/WordLengthException.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.appcatalog.utils; + +/** +* Exception class for checking random word length requirement. +*/ +public class WordLengthException extends Exception { + private static final long serialVersionUID = -217354336577036950L; + + public WordLengthException(String message){ + super(message); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/utils/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/utils/package-info.java new file mode 100644 index 00000000000..f91a0e80da6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/appcatalog/utils/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Application Catalog Utility package. **/ +package org.apache.hadoop.yarn.appcatalog.utils; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index e8286ef93f5..bf188c48539 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -59,8 +59,8 @@ public class ApiServer { private static final Logger LOG = LoggerFactory.getLogger(ApiServer.class); - private static Configuration YARN_CONFIG = new YarnConfiguration(); - private static ServiceClient SERVICE_CLIENT; + private static Configuration yarnConfig = new YarnConfiguration(); + private static ServiceClient serviceClient; static { init(); @@ -68,9 +68,9 @@ // initialize all the common resources - order is important private static void init() { - SERVICE_CLIENT = new ServiceClient(); - SERVICE_CLIENT.init(YARN_CONFIG); - SERVICE_CLIENT.start(); + serviceClient = new ServiceClient(); + serviceClient.init(yarnConfig); + serviceClient.start(); } @GET @@ -91,7 +91,7 @@ public Response createService(Service service) { LOG.info("POST: createService = {}", service); ServiceStatus serviceStatus = new ServiceStatus(); try { - ApplicationId applicationId = SERVICE_CLIENT.actionCreate(service); + ApplicationId applicationId = serviceClient.actionCreate(service); LOG.info("Successfully created service " + service.getName() + " applicationId = " + applicationId); serviceStatus.setState(ACCEPTED); @@ -120,7 +120,7 @@ public Response getService(@PathParam(SERVICE_NAME) String appName) { LOG.info("GET: getService for appName = {}", appName); ServiceStatus serviceStatus = new ServiceStatus(); try { - Service app = SERVICE_CLIENT.getStatus(appName); + Service app = serviceClient.getStatus(appName); return Response.ok(app).build(); } catch (IllegalArgumentException e) { serviceStatus.setDiagnostics(e.getMessage()); @@ -147,9 +147,9 @@ public Response deleteService(@PathParam(SERVICE_NAME) String appName) { private Response stopService(String appName, boolean destroy) { try { - SERVICE_CLIENT.actionStop(appName, destroy); + serviceClient.actionStop(appName, destroy); if (destroy) { - SERVICE_CLIENT.actionDestroy(appName); + serviceClient.actionDestroy(appName); LOG.info("Successfully deleted service {}", appName); } else { LOG.info("Successfully stopped service {}", appName); @@ -184,7 +184,7 @@ public Response updateComponent(@PathParam(SERVICE_NAME) String appName, } ServiceStatus status = new ServiceStatus(); try { - Map original = SERVICE_CLIENT.flexByRestService(appName, + Map original = serviceClient.flexByRestService(appName, Collections.singletonMap(component.getName(), component.getNumberOfContainers())); status.setDiagnostics( @@ -234,9 +234,10 @@ public Response updateService(@PathParam(SERVICE_NAME) String appName, // flex a single component app if (updateServiceData.getNumberOfContainers() != null && !ServiceApiUtil .hasComponent(updateServiceData)) { - Component defaultComp = ServiceApiUtil.createDefaultComponent(updateServiceData); - return updateComponent(updateServiceData.getName(), defaultComp.getName(), - defaultComp); + Component defaultComp = ServiceApiUtil. + createDefaultComponent(updateServiceData); + return updateComponent(updateServiceData.getName(), + defaultComp.getName(), defaultComp); } // If nothing happens consider it a no-op @@ -247,7 +248,7 @@ private Response updateLifetime(String appName, Service updateAppData) { ServiceStatus status = new ServiceStatus(); try { String newLifeTime = - SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime()); + serviceClient.updateLifetime(appName, updateAppData.getLifetime()); status.setDiagnostics( "Service (" + appName + ")'s lifeTime is updated to " + newLifeTime + ", " + updateAppData.getLifetime() @@ -267,7 +268,7 @@ private Response updateLifetime(String appName, Service updateAppData) { private Response startService(String appName) { ServiceStatus status = new ServiceStatus(); try { - SERVICE_CLIENT.actionStart(appName); + serviceClient.actionStart(appName); LOG.info("Successfully started service " + appName); status.setDiagnostics("Service " + appName + " is successfully started."); status.setState(ServiceState.ACCEPTED); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java index fc65a63cc19..980831b9845 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServerWebApp.java @@ -24,6 +24,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.appcatalog.controller.AppCatalogController; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; @@ -48,7 +49,7 @@ * The Java command used to launch this app should call the main method. */ public class ApiServerWebApp extends AbstractService { - private static final Logger logger = LoggerFactory + private static final Logger LOG = LoggerFactory .getLogger(ApiServerWebApp.class); private static final String SEP = ";"; @@ -62,7 +63,7 @@ public static void main(String[] args) throws IOException { apiWebApp.init(new YarnConfiguration()); apiWebApp.serviceStart(); } catch (Exception e) { - logger.error("Got exception starting", e); + LOG.error("Got exception starting", e); apiWebApp.close(); } } @@ -74,8 +75,8 @@ public ApiServerWebApp() { @Override protected void serviceStart() throws Exception { bindAddress = getConfig().getSocketAddr(API_SERVER_ADDRESS, - DEFAULT_API_SERVER_ADDRESS , DEFAULT_API_SERVER_PORT); - logger.info("YARN API server running on " + bindAddress); + DEFAULT_API_SERVER_ADDRESS, DEFAULT_API_SERVER_PORT); + LOG.info("YARN API server running on " + bindAddress); if (UserGroupInformation.isSecurityEnabled()) { doSecureLogin(getConfig()); } @@ -136,25 +137,26 @@ private void startWebApp() throws IOException { String apiPackages = ApiServer.class.getPackage().getName() + SEP + GenericExceptionHandler.class.getPackage().getName() + SEP - + YarnJacksonJaxbJsonProvider.class.getPackage().getName(); + + YarnJacksonJaxbJsonProvider.class.getPackage().getName() + SEP + + AppCatalogController.class.getPackage().getName(); apiServer.addJerseyResourcePackage(apiPackages, "/*"); try { - logger.info("Service starting up. Logging start..."); + LOG.info("Service starting up. Logging start..."); apiServer.start(); - logger.info("Server status = {}", apiServer.toString()); + LOG.info("Server status = {}", apiServer.toString()); for (Configuration conf : apiServer.getWebAppContext() .getConfigurations()) { - logger.info("Configurations = {}", conf); + LOG.info("Configurations = {}", conf); } - logger.info("Context Path = {}", Collections.singletonList( + LOG.info("Context Path = {}", Collections.singletonList( apiServer.getWebAppContext().getContextPath())); - logger.info("ResourceBase = {}", Collections.singletonList( + LOG.info("ResourceBase = {}", Collections.singletonList( apiServer.getWebAppContext().getResourceBase())); - logger.info("War = {}", Collections + LOG.info("War = {}", Collections .singletonList(apiServer.getWebAppContext().getWar())); } catch (Exception ex) { - logger.error("Hadoop HttpServer2 App **failed**", ex); + LOG.error("Hadoop HttpServer2 App **failed**", ex); throw ex; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/package-info.java new file mode 100644 index 00000000000..7b0dba83d82 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** API Service package. **/ +package org.apache.hadoop.yarn.service.webapp; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/application/EmbeddedSolrServerFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/application/EmbeddedSolrServerFactory.java new file mode 100644 index 00000000000..1da79c0b2f4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/application/EmbeddedSolrServerFactory.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.appcatalog.application; + +import org.apache.commons.io.FileUtils; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.core.NodeConfig; +import org.apache.solr.core.SolrResourceLoader; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Embedded Solr Server factory class for running solr unit test cases. + * + */ +public class EmbeddedSolrServerFactory { + + protected EmbeddedSolrServerFactory() { + } + + /** + * Cleans the given solrHome directory and creates a new EmbeddedSolrServer. + * + * @param solrHome + * the Solr home directory to use + * @param configSetHome + * the directory containing config sets + * @param coreName + * the name of the core, must have a matching directory in configHome + * + * @return an EmbeddedSolrServer with a core created for the given coreName + * @throws IOException + */ + public static SolrClient create(final String solrHome, + final String configSetHome, final String coreName) + throws IOException, SolrServerException { + return create(solrHome, configSetHome, coreName, true); + } + + /** + * @param solrHome + * the Solr home directory to use + * @param configSetHome + * the directory containing config sets + * @param coreName + * the name of the core, must have a matching directory in configHome + * @param cleanSolrHome + * if true the directory for solrHome will be deleted and re-created + * if it already exists + * + * @return an EmbeddedSolrServer with a core created for the given coreName + * @throws IOException + */ + public static SolrClient create(final String solrHome, + final String configSetHome, final String coreName, + final boolean cleanSolrHome) throws IOException, SolrServerException { + + final File solrHomeDir = new File(solrHome); + if (solrHomeDir.exists()) { + FileUtils.deleteDirectory(solrHomeDir); + solrHomeDir.mkdirs(); + } else { + solrHomeDir.mkdirs(); + } + + final SolrResourceLoader loader = new SolrResourceLoader( + solrHomeDir.toPath()); + final Path configSetPath = Paths.get(configSetHome).toAbsolutePath(); + + final NodeConfig config = new NodeConfig.NodeConfigBuilder( + "embeddedSolrServerNode", loader) + .setConfigSetBaseDirectory(configSetPath.toString()).build(); + + final EmbeddedSolrServer embeddedSolrServer = new EmbeddedSolrServer(config, + coreName); + + final CoreAdminRequest.Create createRequest = new CoreAdminRequest.Create(); + createRequest.setCoreName(coreName); + createRequest.setConfigSet(coreName); + embeddedSolrServer.request(createRequest); + + return embeddedSolrServer; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestYarnSolrClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestYarnSolrClient.java new file mode 100644 index 00000000000..28b58e433bb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestYarnSolrClient.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.application; + +import org.apache.hadoop.yarn.appcatalog.model.AppCatalogEntry; +import org.apache.hadoop.yarn.appcatalog.model.Application; +import org.apache.solr.client.solrj.SolrClient; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.powermock.api.mockito.PowerMockito; +import static org.powermock.api.mockito.PowerMockito.when; +import static org.powermock.api.support.membermodification.MemberMatcher.method; + +import static org.junit.Assert.*; + +import java.util.List; + +/** + * Test register, search, and recommend applications API. + * + */ +public class TestYarnSolrClient { + + private static final String CONFIGSET_DIR = "src/test/resources/configsets"; + private static SolrClient solrClient; + private static YarnSolrClient spy; + + @Before + public void setup() throws Exception { + String targetLocation = EmbeddedSolrServerFactory.class + .getProtectionDomain().getCodeSource().getLocation().getFile() + "/.."; + + String solrHome = targetLocation + "/solr"; + solrClient = EmbeddedSolrServerFactory.create(solrHome, CONFIGSET_DIR, + "exampleCollection"); + spy = PowerMockito.spy(new YarnSolrClient()); + when(spy, method(YarnSolrClient.class, "getSolrClient")).withNoArguments() + .thenReturn(solrClient); + } + + @After + public void teardown() throws Exception { + try { + solrClient.close(); + } catch (Exception e) { + } + } + + @Test + public void testRegister() throws Exception { + Application example = new Application(); + example.setOrganization("jenkins-ci.org"); + example.setName("jenkins"); + example.setDescription("World leading open source automation system."); + example.setIcon("/css/img/feather.png"); + spy.register(example); + List apps = spy.getRecommendedApps(); + assertEquals(1, apps.size()); + } + + @Test + public void testSearch() throws Exception { + Application example = new Application(); + example.setOrganization("jenkins-ci.org"); + example.setName("jenkins"); + example.setDescription("World leading open source automation system."); + example.setIcon("/css/img/feather.png"); + spy.register(example); + List results = spy.search("name_s:jenkins"); + int expected = 1; + int actual = results.size(); + assertEquals(expected, actual); + } + + @Test + public void testNotFoundSearch() throws Exception { + Application example = new Application(); + example.setOrganization("jenkins-ci.org"); + example.setName("jenkins"); + example.setDescription("World leading open source automation system."); + example.setIcon("/css/img/feather.png"); + spy.register(example); + List results = spy.search("name_s:abc"); + int expected = 0; + int actual = results.size(); + assertEquals(expected, actual); + } + + @Test + public void testGetRecommendedApps() throws Exception { + List expected = spy.getRecommendedApps(); + List actual = spy.getRecommendedApps(); + assertEquals(expected, actual); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppCatalogController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppCatalogController.java new file mode 100644 index 00000000000..34db93ae039 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppCatalogController.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.controller; + +import org.apache.hadoop.yarn.appcatalog.model.AppCatalogEntry; +import org.apache.hadoop.yarn.appcatalog.model.Application; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import javax.ws.rs.Path; +import javax.ws.rs.core.Response; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +import java.util.ArrayList; +import java.util.List; + +/** + * Test application catalog rest api for recommending, searching, + * and register applications. + * + */ +public class TestAppCatalogController { + + private AppCatalogController controller; + + @Before + public void setUp() throws Exception { + this.controller = new AppCatalogController(); + + } + + @Test + public void testGetRecommended() throws Exception { + AppCatalogController ac = Mockito.mock(AppCatalogController.class); + List actual = new ArrayList(); + when(ac.get()).thenReturn(actual); + final List result = ac.get(); + assertEquals(result, actual); + } + + @Test + public void testSearch() throws Exception { + String keyword = "jenkins"; + AppCatalogController ac = Mockito.mock(AppCatalogController.class); + List expected = new ArrayList(); + when(ac.search(keyword)).thenReturn(expected); + final List actual = ac.search(keyword); + assertEquals(expected, actual); + } + + @Test + public void testRegister() throws Exception { + AppCatalogController ac = Mockito.mock(AppCatalogController.class); + Application app = new Application(); + app.setName("jenkins"); + app.setOrganization("jenkins.org"); + app.setDescription("This is a description"); + app.setIcon("/css/img/feather.png"); + Response expected = Response.ok().build(); + when(ac.register(app)).thenReturn(Response.ok().build()); + final Response actual = ac.register(app); + assertEquals(expected.getStatus(), actual.getStatus()); + } + + @Test + public void testPathAnnotation() throws Exception { + assertNotNull(this.controller.getClass().getAnnotations()); + assertThat("The controller has the annotation Path", + this.controller.getClass().isAnnotationPresent(Path.class)); + + final Path path = this.controller.getClass().getAnnotation(Path.class); + assertThat("The path is /ws/v2/app_catalog", path.value(), + is("/ws/v2/app_catalog")); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppDetailsController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppDetailsController.java new file mode 100644 index 00000000000..c017565f439 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppDetailsController.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.controller; + +import org.apache.hadoop.yarn.appcatalog.model.AppEntry; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.Container; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import javax.ws.rs.Path; +import javax.ws.rs.core.Response; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +import java.util.ArrayList; +import java.util.List; + +/** + * Test case for REST API to query deployed application. + * Test includes: + * Getting configuration of deployed application. + * Getting status of deployed application. + * Stop an application. + * Restart an application. + * + */ +public class TestAppDetailsController { + + private AppDetailsController controller; + + @Before + public void setUp() throws Exception { + this.controller = new AppDetailsController(); + + } + + @Test + public void testGetDetails() throws Exception { + String id = "application 1"; + AppDetailsController ac = Mockito.mock(AppDetailsController.class); + + AppEntry actual = new AppEntry(); + actual.setName(id); + when(ac.getDetails(id)).thenReturn(actual); + final AppEntry result = ac.getDetails(id); + assertEquals(result, actual); + } + + @Test + public void testGetStatus() throws Exception { + String id = "application 1"; + AppDetailsController ac = Mockito.mock(AppDetailsController.class); + + Service yarnfile = new Service(); + Component comp = new Component(); + Container c = new Container(); + c.setId("container-1"); + List containers = new ArrayList(); + containers.add(c); + comp.setContainers(containers); + yarnfile.addComponent(comp); + AppEntry actual = new AppEntry(); + actual.setName(id); + actual.setYarnfile(yarnfile); + when(ac.getStatus(id)).thenReturn(actual); + final AppEntry result = ac.getStatus(id); + assertEquals(result, actual); + } + + @Test + public void testStopApp() throws Exception { + String id = "application 1"; + AppDetailsController ac = Mockito.mock(AppDetailsController.class); + + Service yarnfile = new Service(); + Component comp = new Component(); + Container c = new Container(); + c.setId("container-1"); + List containers = new ArrayList(); + containers.add(c); + comp.setContainers(containers); + yarnfile.addComponent(comp); + Response expected = Response.ok().build(); + when(ac.stopApp(id)).thenReturn(Response.ok().build()); + final Response actual = ac.stopApp(id); + assertEquals(expected.getStatus(), actual.getStatus()); + } + + @Test + public void testRestartApp() throws Exception { + String id = "application 1"; + AppDetailsController ac = Mockito.mock(AppDetailsController.class); + + Service yarnfile = new Service(); + Component comp = new Component(); + Container c = new Container(); + c.setId("container-1"); + List containers = new ArrayList(); + containers.add(c); + comp.setContainers(containers); + yarnfile.addComponent(comp); + Response expected = Response.ok().build(); + when(ac.restartApp(id)).thenReturn(Response.ok().build()); + final Response actual = ac.restartApp(id); + assertEquals(expected.getStatus(), actual.getStatus()); + } + + @Test + public void testPathAnnotation() throws Exception { + assertNotNull(this.controller.getClass().getAnnotations()); + assertThat("The controller has the annotation Path", + this.controller.getClass().isAnnotationPresent(Path.class)); + + final Path path = this.controller.getClass().getAnnotation(Path.class); + assertThat("The path is /ws/v2/app_details", path.value(), + is("/ws/v2/app_details")); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppListController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppListController.java new file mode 100644 index 00000000000..f658ccbb6f0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/TestAppListController.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.appcatalog.controller; + +import org.apache.hadoop.yarn.appcatalog.model.AppEntry; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import javax.ws.rs.Path; +import javax.ws.rs.core.Response; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +import java.util.ArrayList; +import java.util.List; + +/** + * Test case for REST API to query deployed applications. + * Test includes: + * Deploy an application + * Delete an application + * Get a list of deployed applications. + * + */ +public class TestAppListController { + + private AppListController controller; + + @Before + public void setUp() throws Exception { + this.controller = new AppListController(); + + } + + @Test + public void testGetList() throws Exception { + AppListController ac = Mockito.mock(AppListController.class); + + List actual = new ArrayList(); + when(ac.getList()).thenReturn(actual); + final List result = ac.getList(); + assertEquals(result, actual); + } + + @Test + public void testDelete() throws Exception { + String id = "application 1"; + AppListController ac = Mockito.mock(AppListController.class); + + Response expected = Response.ok().build(); + when(ac.delete(id, id)).thenReturn(Response.ok().build()); + final Response actual = ac.delete(id, id); + assertEquals(expected.getStatus(), actual.getStatus()); + } + + @Test + public void testDeploy() throws Exception { + String id = "application 1"; + AppListController ac = Mockito.mock(AppListController.class); + + Response expected = Response.ok().build(); + when(ac.deploy(id)).thenReturn(Response.ok().build()); + final Response actual = ac.deploy(id); + assertEquals(expected.getStatus(), actual.getStatus()); + } + + @Test + public void testPathAnnotation() throws Exception { + assertNotNull(this.controller.getClass().getAnnotations()); + assertThat("The controller has the annotation Path", + this.controller.getClass().isAnnotationPresent(Path.class)); + + final Path path = this.controller.getClass().getAnnotation(Path.class); + assertThat("The path is /ws/v2/app_list", path.value(), + is("/ws/v2/app_list")); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt new file mode 100644 index 00000000000..2c164c0b2a1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/lang/stopwords_en.txt @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +# Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +such +that +the +their +then +there +these +they +this +to +was +will +with diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/params.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/params.json new file mode 100644 index 00000000000..06114ef2577 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/params.json @@ -0,0 +1,20 @@ +{"params":{ + "query":{ + "defType":"edismax", + "q.alt":"*:*", + "rows":"10", + "fl":"*,score", + "":{"v":0} + }, + "facets":{ + "facet":"on", + "facet.mincount": "1", + "":{"v":0} + }, + "velocity":{ + "wt": "velocity", + "v.template":"browse", + "v.layout": "layout", + "":{"v":0} + } +}} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/protwords.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/protwords.txt new file mode 100644 index 00000000000..1dfc0abecbf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/schema.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/schema.xml new file mode 100644 index 00000000000..20acbc92b46 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/schema.xml @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml new file mode 100644 index 00000000000..392feec28aa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/solrconfig.xml @@ -0,0 +1,36 @@ + + + + 6.2.1 + + ${solr.data.dir:} + + + + + single + + + + + + + + + + \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/stopwords.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/stopwords.txt new file mode 100644 index 00000000000..ae1e83eeb3d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/stopwords.txt @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/synonyms.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/synonyms.txt new file mode 100644 index 00000000000..7f72128303b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/configsets/exampleCollection/conf/synonyms.txt @@ -0,0 +1,29 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaafoo => aaabar +bbbfoo => bbbfoo bbbbar +cccfoo => cccbar cccbaz +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/log4j.properties new file mode 100644 index 00000000000..81a3f6ad5d2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/log4j.properties @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# log4j configuration used during build and unit tests + +log4j.rootLogger=info,stdout +log4j.threshold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index fb07edc908f..b3d8edcfad3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -22,6 +22,7 @@ 3.1.0-SNAPSHOT hadoop-yarn-services-core + 3.1.0-SNAPSHOT jar Apache Hadoop YARN Services Core diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml index 716fdb70cf0..3b5eaf8a27c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml @@ -24,6 +24,7 @@ 4.0.0 hadoop-yarn-services Apache Hadoop YARN Services + 3.1.0-SNAPSHOT pom diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index 1b3c5f05620..b280d1df57e 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -86,6 +86,10 @@ org.apache.hadoop hadoop-yarn-services-core + + org.apache.hadoop + hadoop-yarn-services-api +