diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 61e9622..6724646 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -880,11 +880,6 @@
${jackson2.version}
- com.fasterxml.jackson.jaxrs
- jackson-jaxrs-json-provider
- ${jackson2.version}
-
-
com.fasterxml.jackson.dataformat
jackson-dataformat-cbor
${jackson2.version}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
index 52a9de6..e1bddb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
@@ -30,7 +30,7 @@
import org.apache.hadoop.yarn.services.api.impl.ApplicationApiService;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.mortbay.jetty.webapp.Configuration;
+import org.eclipse.jetty.webapp.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index a2c67c0..66e9ee9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -225,7 +225,7 @@
javax.servlet
- servlet-api
+ javax.servlet-api
@@ -289,21 +289,6 @@
- org.mortbay.jetty
- jetty
-
-
-
- org.mortbay.jetty
- jetty-util
-
-
-
- org.mortbay.jetty
- jetty-sslengine
-
-
-
javax.servlet.jsp
jsp-api
runtime
@@ -315,12 +300,6 @@
- org.mortbay.jetty
- jetty-sslengine
- 6.1.26
-
-
-
org.yaml
snakeyaml
1.16
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 713cd02..b101d34 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -70,7 +70,6 @@
import org.apache.slider.core.launch.ClasspathConstructor;
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.providers.agent.AgentKeys;
-import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.server.services.utility.PatternValidator;
import org.apache.slider.server.services.workflow.ForkedProcessService;
import org.apache.zookeeper.server.util.KerberosUtil;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 19fa07b..00fc606 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -19,17 +19,17 @@
package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.registry.client.types.AddressTypes;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
-import org.apache.hadoop.registry.client.types.AddressTypes;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
@@ -44,7 +44,6 @@
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.services.workflow.ForkedProcessService;
import org.apache.slider.server.services.workflow.ServiceParent;
import org.apache.slider.server.services.workflow.WorkflowSequenceService;
@@ -77,7 +76,6 @@
private static final Logger log =
LoggerFactory.getLogger(AbstractProviderService.class);
protected StateAccessForProviders amState;
- protected AgentRestOperations restOps;
protected URL amWebAPI;
protected YarnRegistryViewForProviders yarnRegistry;
protected QueueAccess queueAccess;
@@ -127,18 +125,9 @@ public YarnRegistryViewForProviders getYarnRegistry() {
}
@Override
- public AgentRestOperations getAgentRestOperations() {
- return restOps;
- }
-
- @Override
public void notifyContainerCompleted(ContainerId containerId) {
}
- public void setAgentRestOperations(AgentRestOperations agentRestOperations) {
- this.restOps = agentRestOperations;
- }
-
/**
* Load default Configuration
* @param confDir configuration directory
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index b62510a..4ca9326 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -20,10 +20,10 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -37,7 +37,6 @@
import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import java.io.File;
@@ -176,12 +175,6 @@ public void bind(StateAccessForProviders stateAccessor,
void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry);
/**
- * Returns the agent rest operations interface.
- * @return the interface if available, null otherwise.
- */
- AgentRestOperations getAgentRestOperations();
-
- /**
* Build up the endpoint details for this service
* @param details
*/
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
deleted file mode 100644
index 7ca469f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ /dev/null
@@ -1,713 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import com.google.common.io.Files;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.client.ClientUtils;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.AbstractLauncher;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.providers.AbstractClientProvider;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.providers.ProviderUtils;
-import org.apache.slider.providers.agent.application.metadata.Application;
-import org.apache.slider.providers.agent.application.metadata.Component;
-import org.apache.slider.providers.agent.application.metadata.ConfigFile;
-import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
-import org.apache.slider.providers.agent.application.metadata.OSPackage;
-import org.apache.slider.providers.agent.application.metadata.OSSpecific;
-import org.apache.slider.providers.agent.application.metadata.Package;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.Charset;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-
-import static org.apache.slider.common.tools.SliderUtils.getApplicationDefinitionPath;
-
-/** This class implements the client-side aspects of the agent deployer */
-public class AgentClientProvider extends AbstractClientProvider
- implements AgentKeys, SliderKeys {
-
-
- protected static final Logger log =
- LoggerFactory.getLogger(AgentClientProvider.class);
- protected static final String NAME = "agent";
- private static final ProviderUtils providerUtils = new ProviderUtils(log);
- public static final String E_COULD_NOT_READ_METAINFO
- = "Not a valid app package. Could not read metainfo.";
-
- protected Map metaInfoMap = new ConcurrentHashMap();
-
- protected AgentClientProvider(Configuration conf) {
- super(conf);
- }
-
- @Override
- public String getName() {
- return NAME;
- }
-
- @Override
- public List getRoles() {
- return AgentRoles.getRoles();
- }
-
- @Override //Client
- public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem,
- String clustername,
- Configuration configuration,
- AggregateConf instanceDefinition,
- Path clusterDirPath,
- Path generatedConfDirPath,
- boolean secure) throws
- SliderException,
- IOException {
- super.preflightValidateClusterConfiguration(sliderFileSystem, clustername,
- configuration,
- instanceDefinition,
- clusterDirPath,
- generatedConfDirPath, secure);
-
- String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
- .getAppConfOperations());
- Path appDefPath = new Path(appDef);
- sliderFileSystem.verifyFileExists(appDefPath);
-
- String agentConf = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getOption(AGENT_CONF, "");
- if (StringUtils.isNotEmpty(agentConf)) {
- sliderFileSystem.verifyFileExists(new Path(agentConf));
- }
-
- String appHome = instanceDefinition.getAppConfOperations().
- getGlobalOptions().get(PACKAGE_PATH);
- if (SliderUtils.isUnset(appHome)) {
- String agentImage = instanceDefinition.getInternalOperations().
- get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
- sliderFileSystem.verifyFileExists(new Path(agentImage));
- }
- }
-
- @Override
- public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
- SliderException {
- super.validateInstanceDefinition(instanceDefinition, fs);
- log.debug("Validating conf {}", instanceDefinition);
- ConfTreeOperations resources =
- instanceDefinition.getResourceOperations();
-
- providerUtils.validateNodeCount(instanceDefinition, ROLE_NODE,
- 0, -1);
-
- String appDef = null;
- try {
- // Validate the app definition
- appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
- .getAppConfOperations());
- } catch (BadConfigException bce) {
- throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
- }
-
- log.info("Validating app definition {}", appDef);
- String extension = appDef.substring(appDef.lastIndexOf(".") + 1, appDef.length());
- if (!"zip".equals(extension.toLowerCase(Locale.ENGLISH))) {
- throw new BadConfigException("App definition must be packaged as a .zip file. File provided is " + appDef);
- }
-
- Set names = resources.getComponentNames();
- names.remove(COMPONENT_AM);
- Map priorityMap = new HashMap();
-
- for (String name : names) {
- try {
- // Validate the app definition
- appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
- .getAppConfOperations(), name);
- } catch (BadConfigException bce) {
- throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
- }
- Metainfo metaInfo = getMetainfo(fs, appDef);
-
- MapOperations component = resources.getMandatoryComponent(name);
-
- if (metaInfo != null) {
- Component componentDef = metaInfo.getApplicationComponent(
- AgentUtils.getMetainfoComponentName(name,
- instanceDefinition.getAppConfOperations()));
- if (componentDef == null) {
- throw new BadConfigException(
- "Component %s is not a member of application.", name);
- }
- }
-
- int priority =
- component.getMandatoryOptionInt(ResourceKeys.COMPONENT_PRIORITY);
- if (priority <= 0) {
- throw new BadConfigException("Component %s %s value out of range %d",
- name,
- ResourceKeys.COMPONENT_PRIORITY,
- priority);
- }
-
- String existing = priorityMap.get(priority);
- if (existing != null) {
- throw new BadConfigException(
- "Component %s has a %s value %d which duplicates that of %s",
- name,
- ResourceKeys.COMPONENT_PRIORITY,
- priority,
- existing);
- }
- priorityMap.put(priority, name);
-
- // fileSystem may be null for tests
- if (metaInfo != null) {
- Component componentDef = metaInfo.getApplicationComponent(
- AgentUtils.getMetainfoComponentName(name,
- instanceDefinition.getAppConfOperations()));
-
- // ensure that intance count is 0 for client components
- if ("CLIENT".equals(componentDef.getCategory())) {
- MapOperations componentConfig = resources.getMandatoryComponent(name);
- int count =
- componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
- if (count > 0) {
- throw new BadConfigException("Component %s is of type CLIENT and cannot be instantiated."
- + " Use \"slider client install ...\" command instead.",
- name);
- }
- } else {
- MapOperations componentConfig = resources.getMandatoryComponent(name);
- int count =
- componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
- int definedMinCount = componentDef.getMinInstanceCountInt();
- int definedMaxCount = componentDef.getMaxInstanceCountInt();
- if (count < definedMinCount || count > definedMaxCount) {
- throw new BadConfigException("Component %s, %s value %d out of range. "
- + "Expected minimum is %d and maximum is %d",
- name,
- ResourceKeys.COMPONENT_INSTANCES,
- count,
- definedMinCount,
- definedMaxCount);
- }
- }
- }
- }
- }
-
-
- @Override
- public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
- Configuration serviceConf,
- AbstractLauncher launcher,
- AggregateConf instanceDefinition,
- Path snapshotConfDirPath,
- Path generatedConfDirPath,
- Configuration clientConfExtras,
- String libdir,
- Path tempPath,
- boolean miniClusterTestRun) throws
- IOException,
- SliderException {
- String agentImage = instanceDefinition.getInternalOperations().
- get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
- if (SliderUtils.isUnset(agentImage)) {
- Path agentPath = new Path(tempPath.getParent(), PROVIDER_AGENT);
- log.info("Automatically uploading the agent tarball at {}", agentPath);
- fileSystem.getFileSystem().mkdirs(agentPath);
- if (ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) {
- instanceDefinition.getInternalOperations().set(
- InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH,
- new Path(agentPath, AGENT_TAR).toUri());
- }
- }
- }
-
- @Override
- public Set getApplicationTags(SliderFileSystem fileSystem,
- ConfTreeOperations appConf) throws SliderException {
- return getApplicationTags(fileSystem,
- getApplicationDefinitionPath(appConf));
- }
-
- public Set getApplicationTags(SliderFileSystem fileSystem,
- String appDef) throws SliderException {
- Set tags;
- Metainfo metaInfo = getMetainfo(fileSystem, appDef);
-
- if (metaInfo == null) {
- log.error("Error retrieving metainfo from {}", appDef);
- throw new SliderException("Error parsing metainfo file, possibly bad structure.");
- }
-
- Application application = metaInfo.getApplication();
- tags = new HashSet();
- tags.add("Name: " + application.getName());
- tags.add("Version: " + application.getVersion());
- tags.add("Description: " + SliderUtils.truncate(application.getComment(), 80));
-
- return tags;
- }
-
- @Override
- public void processClientOperation(SliderFileSystem fileSystem,
- RegistryOperations rops,
- Configuration configuration,
- String operation,
- File clientInstallPath,
- File appPackage,
- JSONObject config,
- String name) throws SliderException {
- // create temp folder
- // create sub-folders app_pkg, agent_pkg, command
- File tmpDir = Files.createTempDir();
- log.info("Command is being executed at {}", tmpDir.getAbsolutePath());
- File appPkgDir = new File(tmpDir, "app_pkg");
- appPkgDir.mkdir();
-
- File agentPkgDir = new File(tmpDir, "agent_pkg");
- agentPkgDir.mkdir();
-
- File cmdDir = new File(tmpDir, "command");
- cmdDir.mkdir();
-
- Metainfo metaInfo = null;
- JSONObject defaultConfig = null;
- try {
- // expand app package into /app_pkg
- ZipInputStream zipInputStream = null;
- try {
- zipInputStream = new ZipInputStream(new FileInputStream(appPackage));
- {
- ZipEntry zipEntry = zipInputStream.getNextEntry();
- while (zipEntry != null) {
- log.info("Processing {}", zipEntry.getName());
- String filePath = appPkgDir + File.separator + zipEntry.getName();
- if (!zipEntry.isDirectory()) {
- log.info("Extracting file {}", filePath);
- extractFile(zipInputStream, filePath);
-
- if ("metainfo.xml".equals(zipEntry.getName())) {
- FileInputStream input = null;
- try {
- input = new FileInputStream(filePath);
- metaInfo = new MetainfoParser().fromXmlStream(input);
- } finally {
- IOUtils.closeStream(input);
- }
- } else if ("metainfo.json".equals(zipEntry.getName())) {
- FileInputStream input = null;
- try {
- input = new FileInputStream(filePath);
- metaInfo = new MetainfoParser().fromJsonStream(input);
- } finally {
- IOUtils.closeStream(input);
- }
- } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) {
- try {
- defaultConfig = new JSONObject(FileUtils.readFileToString(new File(filePath), Charset.defaultCharset()));
- } catch (JSONException jex) {
- throw new SliderException("Unable to read default client config.", jex);
- }
- }
- } else {
- log.info("Creating dir {}", filePath);
- File dir = new File(filePath);
- dir.mkdir();
- }
- zipInputStream.closeEntry();
- zipEntry = zipInputStream.getNextEntry();
- }
- }
- } finally {
- zipInputStream.close();
- }
-
- if (metaInfo == null) {
- throw new BadConfigException(E_COULD_NOT_READ_METAINFO);
- }
-
- String clientScript = null;
- String clientComponent = null;
- for (Component component : metaInfo.getApplication().getComponents()) {
- if (component.getCategory().equals("CLIENT")) {
- clientComponent = component.getName();
- if (component.getCommandScript() != null) {
- clientScript = component.getCommandScript().getScript();
- }
- break;
- }
- }
-
- if (SliderUtils.isUnset(clientScript)) {
- log.info("Installing CLIENT without script");
- List packages = metaInfo.getApplication().getPackages();
- if (packages.size() > 0) {
- // retrieve package resources from HDFS and extract
- for (Package pkg : packages) {
- Path pkgPath = fileSystem.buildResourcePath(pkg.getName());
- if (!fileSystem.isFile(pkgPath) && name != null) {
- pkgPath = fileSystem.buildResourcePath(name, pkg.getName());
- }
- if (!fileSystem.isFile(pkgPath)) {
- throw new IOException("Package doesn't exist as a resource: " +
- pkg.getName());
- }
- if ("archive".equals(pkg.getType())) {
- File pkgFile = new File(tmpDir, pkg.getName());
- fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile);
- expandTar(pkgFile, clientInstallPath);
- } else {
- File pkgFile = new File(clientInstallPath, pkg.getName());
- fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile);
- }
- }
- } else {
- // extract tarball from app def
- for (OSSpecific osSpecific : metaInfo.getApplication()
- .getOSSpecifics()) {
- for (OSPackage pkg : osSpecific.getPackages()) {
- if ("tarball".equals(pkg.getType())) {
- File pkgFile = new File(appPkgDir, pkg.getName());
- expandTar(pkgFile, clientInstallPath);
- }
- }
- }
- }
- if (name == null) {
- log.warn("Conf files not being generated because no app name was " +
- "provided");
- return;
- }
- File confInstallDir;
- String clientRoot = null;
- if (config != null) {
- try {
- clientRoot = config.getJSONObject("global")
- .getString(APP_CLIENT_ROOT);
- } catch (JSONException e) {
- log.info("Couldn't read {} from provided client config, falling " +
- "back on default", APP_CLIENT_ROOT);
- }
- }
- if (clientRoot == null && defaultConfig != null) {
- try {
- clientRoot = defaultConfig.getJSONObject("global")
- .getString(APP_CLIENT_ROOT);
- } catch (JSONException e) {
- log.info("Couldn't read {} from default client config, using {}",
- APP_CLIENT_ROOT, clientInstallPath);
- }
- }
- if (clientRoot == null) {
- confInstallDir = clientInstallPath;
- } else {
- confInstallDir = new File(new File(clientInstallPath, clientRoot), "conf");
- if (!confInstallDir.exists()) {
- confInstallDir.mkdirs();
- }
- }
- String user = RegistryUtils.currentUser();
- for (ConfigFile configFile : metaInfo.getComponentConfigFiles(clientComponent)) {
- retrieveConfigFile(rops, configuration, configFile, name, user,
- confInstallDir);
- }
- } else {
- log.info("Installing CLIENT using script {}", clientScript);
- expandAgentTar(agentPkgDir);
-
- JSONObject commandJson = getCommandJson(defaultConfig, config, metaInfo, clientInstallPath, name);
- FileWriter file = new FileWriter(new File(cmdDir, "command.json"));
- try {
- file.write(commandJson.toString());
-
- } catch (IOException e) {
- log.error("Couldn't write command.json to file");
- } finally {
- file.flush();
- file.close();
- }
-
- runCommand(appPkgDir, agentPkgDir, cmdDir, clientScript);
- }
-
- } catch (IOException ioex) {
- log.warn("Error while executing INSTALL command {}", ioex.getMessage());
- throw new SliderException("INSTALL client failed.");
- }
- }
-
- protected void runCommand(
- File appPkgDir,
- File agentPkgDir,
- File cmdDir,
- String clientScript) throws SliderException {
- int exitCode = 0;
- Exception exp = null;
- try {
- String clientScriptPath = appPkgDir.getAbsolutePath() + File.separator + "package" +
- File.separator + clientScript;
- List command = Arrays.asList(PYTHON_EXE,
- "-S",
- clientScriptPath,
- "INSTALL",
- cmdDir.getAbsolutePath() + File.separator + "command.json",
- appPkgDir.getAbsolutePath() + File.separator + "package",
- cmdDir.getAbsolutePath() + File.separator + "command-out.json",
- "DEBUG");
- ProcessBuilder pb = new ProcessBuilder(command);
- log.info("Command: " + StringUtils.join(pb.command(), " "));
- pb.environment().put(PYTHONPATH,
- agentPkgDir.getAbsolutePath()
- + File.separator + "slider-agent" + File.pathSeparator
- + agentPkgDir.getAbsolutePath()
- + File.separator + "slider-agent/jinja2");
- log.info("{}={}", PYTHONPATH, pb.environment().get(PYTHONPATH));
-
- Process proc = pb.start();
- InputStream stderr = proc.getErrorStream();
- InputStream stdout = proc.getInputStream();
- BufferedReader stdOutReader = new BufferedReader(new InputStreamReader(stdout));
- BufferedReader stdErrReader = new BufferedReader(new InputStreamReader(stderr));
-
- proc.waitFor();
-
- String line;
- while ((line = stdOutReader.readLine()) != null) {
- log.info("Stdout: " + line);
- }
- while ((line = stdErrReader.readLine()) != null) {
- log.info("Stderr: " + line);
- }
-
- exitCode = proc.exitValue();
- log.info("Exit value is {}", exitCode);
- } catch (IOException e) {
- exp = e;
- } catch (InterruptedException e) {
- exp = e;
- }
-
- if (exitCode != 0) {
- throw new SliderException("INSTALL client failed with exit code " + exitCode);
- }
-
- if (exp != null) {
- log.error("Error while executing INSTALL command {}. Stack trace {}",
- exp.getMessage(),
- ExceptionUtils.getStackTrace(exp));
- throw new SliderException("INSTALL client failed.", exp);
- }
- }
-
- private void expandAgentTar(File agentPkgDir) throws IOException {
- String libDirProp = SliderUtils.getLibDir();
- File tarFile = new File(libDirProp, AGENT_TAR);
- expandTar(tarFile, agentPkgDir);
- }
-
- private void expandTar(File tarFile, File destDir) throws IOException {
- log.info("Expanding tar {} to {}", tarFile, destDir);
- TarArchiveInputStream tarIn = new TarArchiveInputStream(
- new GzipCompressorInputStream(
- new BufferedInputStream(
- new FileInputStream(tarFile)
- )
- )
- );
- try {
- TarArchiveEntry tarEntry = tarIn.getNextTarEntry();
- while (tarEntry != null) {
- File destPath = new File(destDir, tarEntry.getName());
- File parent = destPath.getParentFile();
- if (!parent.exists()) {
- parent.mkdirs();
- }
- if (tarEntry.isDirectory()) {
- destPath.mkdirs();
- } else {
- byte[] byteToRead = new byte[1024];
- BufferedOutputStream buffOut =
- new BufferedOutputStream(new FileOutputStream(destPath));
- try {
- int len;
- while ((len = tarIn.read(byteToRead)) != -1) {
- buffOut.write(byteToRead, 0, len);
- }
- } finally {
- buffOut.close();
- }
- }
- if ((tarEntry.getMode() & 0100) != 0) {
- destPath.setExecutable(true);
- }
- tarEntry = tarIn.getNextTarEntry();
- }
- } finally {
- tarIn.close();
- }
- }
-
- private void retrieveConfigFile(RegistryOperations rops,
- Configuration configuration, ConfigFile configFile, String name,
- String user, File destDir) throws IOException, SliderException {
- log.info("Retrieving config {} to {}", configFile.getDictionaryName(),
- destDir);
- PublishedConfiguration published = ClientUtils.getConfigFromRegistry(rops,
- configuration, configFile.getDictionaryName(), name, user, true);
- ClientUtils.saveOrReturnConfig(published, configFile.getType(),
- destDir, configFile.getFileName());
- }
-
- protected JSONObject getCommandJson(JSONObject defaultConfig,
- JSONObject inputConfig,
- Metainfo metainfo,
- File clientInstallPath,
- String name) throws SliderException {
- try {
- JSONObject pkgList = new JSONObject();
- pkgList.put(PACKAGE_LIST,
- AgentProviderService.getPackageListFromApplication(metainfo.getApplication()));
- JSONObject obj = new JSONObject();
- obj.put("hostLevelParams", pkgList);
-
- String user = RegistryUtils.currentUser();
- JSONObject configuration = new JSONObject();
- JSONObject global = new JSONObject();
- global.put("app_install_dir", clientInstallPath.getAbsolutePath());
- global.put("app_user", user);
- if (name != null) {
- global.put("app_name", name);
- }
-
- if (defaultConfig != null) {
- readConfigEntries(defaultConfig, clientInstallPath, global, name, user);
- }
- if (inputConfig != null) {
- readConfigEntries(inputConfig, clientInstallPath, global, name, user);
- }
-
- configuration.put("global", global);
- obj.put("configurations", configuration);
- return obj;
- } catch (JSONException jex) {
- log.warn("Error while executing INSTALL command {}", jex.getMessage());
- throw new SliderException("INSTALL client failed.");
- }
- }
-
- private void readConfigEntries(JSONObject inpConfig,
- File clientInstallPath,
- JSONObject globalConfig,
- String name, String user)
- throws JSONException {
- JSONObject globalSection = inpConfig.getJSONObject("global");
- Iterator it = globalSection.keys();
- while (it.hasNext()) {
- String key = (String) it.next();
- String value = globalSection.getString(key);
- if (SliderUtils.isSet(value)) {
- value = value.replace("{app_install_dir}", clientInstallPath.getAbsolutePath());
- value = value.replace("{app_user}", user);
- if (name != null) {
- value = value.replace("{app_name}", name);
- }
- }
- if (globalConfig.has(key)) {
- // last one wins
- globalConfig.remove(key);
- }
- globalConfig.put(key, value);
- }
- }
-
- private void extractFile(ZipInputStream zipInputStream, String filePath) throws IOException {
- BufferedOutputStream output = new BufferedOutputStream(new FileOutputStream(filePath));
- try {
- byte[] bytesRead = new byte[4096];
- int read = 0;
- while ((read = zipInputStream.read(bytesRead)) != -1) {
- output.write(bytesRead, 0, read);
- }
- } finally {
- output.close();
- }
- }
-
- private Metainfo getMetainfo(SliderFileSystem fs, String appDef) {
- Metainfo metaInfo = metaInfoMap.get(appDef);
- if (fs != null && metaInfo == null) {
- try {
- metaInfo = AgentUtils.getApplicationMetainfo(fs, appDef, false);
- metaInfoMap.put(appDef, metaInfo);
- } catch (IOException ioe) {
- // Ignore missing metainfo file for now
- log.info("Missing metainfo {}", ioe.getMessage());
- } catch (BadConfigException bce) {
- log.info("Bad Configuration {}", bce.getMessage());
- }
- }
- return metaInfo;
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
deleted file mode 100644
index 18c6374..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.regex.Pattern;
-
-class AgentLaunchParameter {
- public static final Logger log =
- LoggerFactory.getLogger(AgentLaunchParameter.class);
- private static final String DEFAULT_PARAMETER = "";
- private static final String ANY_COMPONENT = "ANY";
- private static final String NONE_VALUE = "NONE";
- private final Map launchParameterTracker;
-
- public AgentLaunchParameter(String parameters) {
- launchParameterTracker = parseExpectedLaunchParameters(parameters);
- }
-
- /**
- * Get command for the component type
- *
- * @param componentGroup
- *
- * @return
- */
- public String getNextLaunchParameter(String componentGroup) {
- if (launchParameterTracker != null) {
- if (launchParameterTracker.containsKey(componentGroup)
- || launchParameterTracker.containsKey(ANY_COMPONENT)) {
- synchronized (this) {
- CommandTracker indexTracker = null;
- if (launchParameterTracker.containsKey(componentGroup)) {
- indexTracker = launchParameterTracker.get(componentGroup);
- } else {
- indexTracker = launchParameterTracker.get(ANY_COMPONENT);
- }
-
- return indexTracker.getNextCommand();
- }
- }
- }
-
- return DEFAULT_PARAMETER;
- }
-
- /**
- * Parse launch parameters of the form ANY:PARAM_FOR_FIRST:PARAM_FOR_SECOND:...:PARAM_FOR_REST|HBASE_MASTER:...
- *
- * E.g. ANY:DO_NOT_REGISTER:DO_NOT_HEARTBEAT:NONE For any container, first one gets DO_NOT_REGISTER second one gets
- * DO_NOT_HEARTBEAT, then all of the rest get nothing
- *
- * E.g. HBASE_MASTER:FAIL_AFTER_START:NONE For HBASE_MASTER, first one gets FAIL_AFTER_START then "" for all
- *
- * @param launchParameters
- *
- * @return
- */
- Map parseExpectedLaunchParameters(String launchParameters) {
- Map trackers = null;
- if (launchParameters != null && launchParameters.length() > 0) {
- String[] componentSpecificParameters = launchParameters.split(Pattern.quote("|"));
- for (String componentSpecificParameter : componentSpecificParameters) {
- if (componentSpecificParameter.length() != 0) {
- String[] parameters = componentSpecificParameter.split(Pattern.quote(":"));
-
- if (parameters.length > 1 && parameters[0].length() > 0) {
-
- for (int index = 1; index < parameters.length; index++) {
- if (parameters[index].equals(NONE_VALUE)) {
- parameters[index] = DEFAULT_PARAMETER;
- }
- }
-
- if (trackers == null) {
- trackers = new HashMap(10);
- }
- String componentName = parameters[0];
- CommandTracker tracker = new CommandTracker(Arrays.copyOfRange(parameters, 1, parameters.length));
- trackers.put(componentName, tracker);
- }
- }
- }
- }
-
- return trackers;
- }
-
- class CommandTracker {
- private final int maxIndex;
- private final String[] launchCommands;
- private int currentIndex;
-
- CommandTracker(String[] launchCommands) {
- this.currentIndex = 0;
- this.maxIndex = launchCommands.length - 1;
- this.launchCommands = launchCommands;
- }
-
- String getNextCommand() {
- String retVal = launchCommands[currentIndex];
- if (currentIndex != maxIndex) {
- currentIndex++;
- }
-
- return retVal;
- }
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java
deleted file mode 100644
index d5ca749..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.slider.providers.AbstractClientProvider;
-import org.apache.slider.providers.ProviderService;
-import org.apache.slider.providers.SliderProviderFactory;
-
-public class AgentProviderFactory extends SliderProviderFactory {
-
- public static final String CLASSNAME =
- "org.apache.slider.providers.agent.AgentProviderFactory";
-
- public AgentProviderFactory() {
- }
-
- public AgentProviderFactory(Configuration conf) {
- super(conf);
- }
-
- @Override
- public AbstractClientProvider createClientProvider() {
- return new AgentClientProvider(getConf());
- }
-
- @Override
- public ProviderService createServerProvider() {
- return new AgentProviderService();
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
deleted file mode 100644
index 499812e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ /dev/null
@@ -1,2850 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ProtocolTypes;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.slider.api.ClusterDescription;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.OptionKeys;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.StatusKeys;
-import org.apache.slider.common.SliderExitCodes;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.NoSuchNodeException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.CommandLineBuilder;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.ConfigUtils;
-import org.apache.slider.core.registry.docstore.ExportEntry;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.docstore.PublishedExports;
-import org.apache.slider.core.registry.info.CustomRegistryConstants;
-import org.apache.slider.providers.AbstractProviderService;
-import org.apache.slider.providers.MonitorDetail;
-import org.apache.slider.providers.ProviderCore;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.providers.ProviderUtils;
-import org.apache.slider.providers.agent.application.metadata.AbstractComponent;
-import org.apache.slider.providers.agent.application.metadata.Application;
-import org.apache.slider.providers.agent.application.metadata.CommandOrder;
-import org.apache.slider.providers.agent.application.metadata.CommandScript;
-import org.apache.slider.providers.agent.application.metadata.Component;
-import org.apache.slider.providers.agent.application.metadata.ComponentCommand;
-import org.apache.slider.providers.agent.application.metadata.ComponentExport;
-import org.apache.slider.providers.agent.application.metadata.ComponentsInAddonPackage;
-import org.apache.slider.providers.agent.application.metadata.ConfigFile;
-import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
-import org.apache.slider.providers.agent.application.metadata.DockerContainer;
-import org.apache.slider.providers.agent.application.metadata.Export;
-import org.apache.slider.providers.agent.application.metadata.ExportGroup;
-import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.providers.agent.application.metadata.OSPackage;
-import org.apache.slider.providers.agent.application.metadata.OSSpecific;
-import org.apache.slider.providers.agent.application.metadata.Package;
-import org.apache.slider.providers.agent.application.metadata.PropertyInfo;
-import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
-import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
-import org.apache.slider.server.appmaster.state.ContainerPriority;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
-import org.apache.slider.server.appmaster.web.rest.agent.CommandReport;
-import org.apache.slider.server.appmaster.web.rest.agent.ComponentStatus;
-import org.apache.slider.server.appmaster.web.rest.agent.ExecutionCommand;
-import org.apache.slider.server.appmaster.web.rest.agent.HeartBeat;
-import org.apache.slider.server.appmaster.web.rest.agent.HeartBeatResponse;
-import org.apache.slider.server.appmaster.web.rest.agent.Register;
-import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse;
-import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus;
-import org.apache.slider.server.appmaster.web.rest.agent.StatusCommand;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Scanner;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Pattern;
-
-import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS;
-
-/**
- * This class implements the server-side logic for application deployment through Slider application package
- */
-public class AgentProviderService extends AbstractProviderService implements
- ProviderCore,
- AgentKeys,
- SliderKeys, AgentRestOperations {
-
-
- protected static final Logger log =
- LoggerFactory.getLogger(AgentProviderService.class);
- private static final ProviderUtils providerUtils = new ProviderUtils(log);
- private static final String LABEL_MAKER = "___";
- private static final String CONTAINER_ID = "container_id";
- private static final String GLOBAL_CONFIG_TAG = "global";
- private static final String COMPONENT_TAG = "component";
- private static final String APPLICATION_TAG = "application";
- private static final String COMPONENT_DATA_TAG = "ComponentInstanceData";
- private static final String SHARED_PORT_TAG = "SHARED";
- private static final String PER_CONTAINER_TAG = "{PER_CONTAINER}";
- private static final int MAX_LOG_ENTRIES = 40;
- private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
-
- private final Object syncLock = new Object();
- private final ComponentTagProvider tags = new ComponentTagProvider();
- private int heartbeatMonitorInterval = 0;
- private AgentClientProvider clientProvider;
- private AtomicInteger taskId = new AtomicInteger(0);
- private volatile Map metaInfoMap = new HashMap<>();
- private SliderFileSystem fileSystem = null;
- private Map defaultConfigs = null;
- private ComponentCommandOrder commandOrder = new ComponentCommandOrder();
- private HeartbeatMonitor monitor;
- private Boolean canAnyMasterPublish = null;
- private AgentLaunchParameter agentLaunchParameter = null;
- private String clusterName = null;
- private boolean isInUpgradeMode;
- private Set upgradeContainers = new HashSet();
- private boolean appStopInitiated;
-
- private final Map componentStatuses =
- new ConcurrentHashMap();
- private final Map> componentInstanceData =
- new ConcurrentHashMap>();
- private final Map>> exportGroups =
- new ConcurrentHashMap>>();
- private final Map> allocatedPorts =
- new ConcurrentHashMap>();
- private final Map packageMetainfo =
- new ConcurrentHashMap();
-
- private final Map logFolderExports =
- Collections.synchronizedMap(new LinkedHashMap(MAX_LOG_ENTRIES, 0.75f, false) {
- protected boolean removeEldestEntry(Map.Entry eldest) {
- return size() > MAX_LOG_ENTRIES;
- }
- });
- private final Map workFolderExports =
- Collections.synchronizedMap(new LinkedHashMap(MAX_LOG_ENTRIES, 0.75f, false) {
- protected boolean removeEldestEntry(Map.Entry eldest) {
- return size() > MAX_LOG_ENTRIES;
- }
- });
- private final Map> containerExportsMap =
- new HashMap>();
-
- private static class MetainfoHolder {
- Metainfo metaInfo;
- private Map defaultConfigs = null;
-
- public MetainfoHolder(Metainfo metaInfo,
- Map defaultConfigs) {
- this.metaInfo = metaInfo;
- this.defaultConfigs = defaultConfigs;
- }
- }
-
- /**
- * Create an instance of AgentProviderService
- */
- public AgentProviderService() {
- super("AgentProviderService");
- setAgentRestOperations(this);
- setHeartbeatMonitorInterval(DEFAULT_HEARTBEAT_MONITOR_INTERVAL);
- }
-
- @Override
- public String getHumanName() {
- return "Slider Agent";
- }
-
- @Override
- public List getRoles() {
- return AgentRoles.getRoles();
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- clientProvider = new AgentClientProvider(conf);
- }
-
- @Override
- public void validateInstanceDefinition(AggregateConf instanceDefinition)
- throws
- SliderException {
- clientProvider.validateInstanceDefinition(instanceDefinition, null);
-
- ConfTreeOperations resources =
- instanceDefinition.getResourceOperations();
-
- Set names = resources.getComponentNames();
- names.remove(COMPONENT_AM);
- for (String name : names) {
- Component componentDef = getApplicationComponent(name);
- if (componentDef == null) {
- // component member is validated elsewhere, so we don't need to throw
- // an exception here
- continue;
- }
-
- MapOperations componentConfig = resources.getMandatoryComponent(name);
- int count =
- componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
- int definedMinCount = componentDef.getMinInstanceCountInt();
- int definedMaxCount = componentDef.getMaxInstanceCountInt();
- if (count < definedMinCount || count > definedMaxCount) {
- throw new BadConfigException("Component %s, %s value %d out of range. "
- + "Expected minimum is %d and maximum is %d",
- name,
- ResourceKeys.COMPONENT_INSTANCES,
- count,
- definedMinCount,
- definedMaxCount);
- }
- }
- }
-
- // Reads the metainfo.xml in the application package and loads it
- private void buildMetainfo(AggregateConf instanceDefinition,
- SliderFileSystem fileSystem,
- String roleGroup)
- throws IOException, SliderException {
- String mapKey = instanceDefinition.getAppConfOperations()
- .getComponentOpt(roleGroup, ROLE_PREFIX, DEFAULT_METAINFO_MAP_KEY);
- String appDef = SliderUtils.getApplicationDefinitionPath(
- instanceDefinition.getAppConfOperations(), roleGroup);
- MapOperations component = null;
- if (roleGroup != null) {
- component = instanceDefinition.getAppConfOperations().getComponent(roleGroup);
- }
-
- MetainfoHolder metaInfoHolder = metaInfoMap.get(mapKey);
- if (metaInfoHolder == null) {
- synchronized (syncLock) {
- if (this.fileSystem == null) {
- this.fileSystem = fileSystem;
- }
- metaInfoHolder = metaInfoMap.get(mapKey);
- if (metaInfoHolder == null) {
- readAndSetHeartbeatMonitoringInterval(instanceDefinition);
- initializeAgentDebugCommands(instanceDefinition);
-
- Metainfo metaInfo = getApplicationMetainfo(fileSystem, appDef, false);
- log.info("Master package metainfo: {}", metaInfo.toString());
- if (metaInfo == null || metaInfo.getApplication() == null) {
- log.error("metainfo.xml is unavailable or malformed at {}.", appDef);
- throw new SliderException(
- "metainfo.xml is required in app package.");
- }
- List commandOrders = metaInfo.getApplication()
- .getCommandOrders();
- if (!DEFAULT_METAINFO_MAP_KEY.equals(mapKey)) {
- for (Component comp : metaInfo.getApplication().getComponents()) {
- comp.setName(mapKey + comp.getName());
- log.info("Modifying external metainfo component name to {}",
- comp.getName());
- }
- for (CommandOrder co : commandOrders) {
- log.info("Adding prefix {} to command order {}",
- mapKey, co);
- co.setCommand(mapKey + co.getCommand());
- co.setRequires(mapKey + co.getRequires());
- }
- }
- log.debug("Merging command orders {} for {}", commandOrders,
- roleGroup);
- commandOrder.mergeCommandOrders(commandOrders,
- instanceDefinition.getResourceOperations());
- Map defaultConfigs =
- initializeDefaultConfigs(fileSystem, appDef, metaInfo);
- metaInfoMap.put(mapKey, new MetainfoHolder(metaInfo, defaultConfigs));
- monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval());
- monitor.start();
-
- // build a map from component to metainfo
- String addonAppDefString = instanceDefinition.getAppConfOperations()
- .getGlobalOptions().getOption(ADDONS, null);
- if (component != null) {
- addonAppDefString = component.getOption(ADDONS, addonAppDefString);
- }
- log.debug("All addon appdefs: {}", addonAppDefString);
- if (addonAppDefString != null) {
- Scanner scanner = new Scanner(addonAppDefString).useDelimiter(",");
- while (scanner.hasNext()) {
- String addonAppDef = scanner.next();
- String addonAppDefPath = instanceDefinition
- .getAppConfOperations().getGlobalOptions().get(addonAppDef);
- if (component != null) {
- addonAppDefPath = component.getOption(addonAppDef, addonAppDefPath);
- }
- log.debug("Addon package {} is stored at: {}", addonAppDef
- + addonAppDefPath);
- Metainfo addonMetaInfo = getApplicationMetainfo(fileSystem,
- addonAppDefPath, true);
- addonMetaInfo.validate();
- packageMetainfo.put(addonMetaInfo.getApplicationPackage()
- .getName(), addonMetaInfo);
- }
- log.info("Metainfo map for master and addon: {}",
- packageMetainfo.toString());
- }
- }
- }
- }
- }
-
- @Override
- public void initializeApplicationConfiguration(
- AggregateConf instanceDefinition, SliderFileSystem fileSystem,
- String roleGroup)
- throws IOException, SliderException {
- buildMetainfo(instanceDefinition, fileSystem, roleGroup);
- }
-
- @Override
- public void buildContainerLaunchContext(ContainerLauncher launcher,
- AggregateConf instanceDefinition,
- Container container,
- ProviderRole providerRole,
- SliderFileSystem fileSystem,
- Path generatedConfPath,
- MapOperations resourceComponent,
- MapOperations appComponent,
- Path containerTmpDirPath) throws
- IOException,
- SliderException {
-
- String roleName = providerRole.name;
- String roleGroup = providerRole.group;
- String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
- .getAppConfOperations(), roleGroup);
-
- initializeApplicationConfiguration(instanceDefinition, fileSystem, roleGroup);
-
- log.info("Build launch context for Agent");
- log.debug(instanceDefinition.toString());
-
- //if we are launching docker based app on yarn, then we need to pass docker image
- if (isYarnDockerContainer(roleGroup)) {
- launcher.setYarnDockerMode(true);
- launcher.setDockerImage(getConfigFromMetaInfo(roleGroup, "image"));
- launcher.setDockerNetwork(getConfigFromMetaInfo(roleGroup, "network"));
- launcher.setRunPrivilegedContainer(getConfigFromMetaInfo(roleGroup, "runPriviledgedContainer"));
- launcher
- .setYarnContainerMountPoints(getConfigFromMetaInfoWithAppConfigOverriding(
- roleGroup, "yarn.container.mount.points"));
- }
-
- // Set the environment
- launcher.putEnv(SliderUtils.buildEnvMap(appComponent,
- providerUtils.getStandardTokenMap(getAmState().getAppConfSnapshot(),
- getAmState().getInternalsSnapshot(), roleName, roleGroup,
- getClusterName())));
-
- String workDir = ApplicationConstants.Environment.PWD.$();
- launcher.setEnv("AGENT_WORK_ROOT", workDir);
- log.info("AGENT_WORK_ROOT set to {}", workDir);
- String logDir = ApplicationConstants.LOG_DIR_EXPANSION_VAR;
- launcher.setEnv("AGENT_LOG_ROOT", logDir);
- log.info("AGENT_LOG_ROOT set to {}", logDir);
- if (System.getenv(HADOOP_USER_NAME) != null) {
- launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
- }
- // for 2-Way SSL
- launcher.setEnv(SLIDER_PASSPHRASE, instanceDefinition.getPassphrase());
- //add english env
- launcher.setEnv("LANG", "en_US.UTF-8");
- launcher.setEnv("LC_ALL", "en_US.UTF-8");
- launcher.setEnv("LANGUAGE", "en_US.UTF-8");
-
- //local resources
-
- // TODO: Should agent need to support App Home
- String scriptPath = new File(AGENT_MAIN_SCRIPT_ROOT, AGENT_MAIN_SCRIPT).getPath();
- String appHome = instanceDefinition.getAppConfOperations().
- getGlobalOptions().get(PACKAGE_PATH);
- if (SliderUtils.isSet(appHome)) {
- scriptPath = new File(appHome, AGENT_MAIN_SCRIPT).getPath();
- }
-
- // set PYTHONPATH
- List pythonPaths = new ArrayList();
- pythonPaths.add(AGENT_MAIN_SCRIPT_ROOT);
- pythonPaths.add(AGENT_JINJA2_ROOT);
- String pythonPath = StringUtils.join(File.pathSeparator, pythonPaths);
- launcher.setEnv(PYTHONPATH, pythonPath);
- log.info("PYTHONPATH set to {}", pythonPath);
-
- Path agentImagePath = null;
- String agentImage = instanceDefinition.getInternalOperations().
- get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
- if (SliderUtils.isUnset(agentImage)) {
- agentImagePath =
- new Path(new Path(new Path(instanceDefinition.getInternalOperations().get(InternalKeys.INTERNAL_TMP_DIR),
- container.getId().getApplicationAttemptId().getApplicationId().toString()),
- PROVIDER_AGENT),
- AGENT_TAR);
- } else {
- agentImagePath = new Path(agentImage);
- }
-
- if (fileSystem.getFileSystem().exists(agentImagePath)) {
- LocalResource agentImageRes = fileSystem.createAmResource(agentImagePath, LocalResourceType.ARCHIVE);
- launcher.addLocalResource(AGENT_INSTALL_DIR, agentImageRes);
- } else {
- String msg =
- String.format("Required agent image slider-agent.tar.gz is unavailable at %s", agentImagePath.toString());
- MapOperations compOps = appComponent;
- boolean relaxVerificationForTest = compOps != null ? Boolean.valueOf(compOps.
- getOptionBool(TEST_RELAX_VERIFICATION, false)) : false;
- log.error(msg);
-
- if (!relaxVerificationForTest) {
- throw new SliderException(SliderExitCodes.EXIT_DEPLOYMENT_FAILED, msg);
- }
- }
-
- log.info("Using {} for agent.", scriptPath);
- LocalResource appDefRes = fileSystem.createAmResource(
- fileSystem.getFileSystem().resolvePath(new Path(appDef)),
- LocalResourceType.ARCHIVE);
- launcher.addLocalResource(APP_DEFINITION_DIR, appDefRes);
-
- for (Package pkg : getMetaInfo(roleGroup).getApplication().getPackages()) {
- Path pkgPath = fileSystem.buildResourcePath(pkg.getName());
- if (!fileSystem.isFile(pkgPath)) {
- pkgPath = fileSystem.buildResourcePath(getClusterName(),
- pkg.getName());
- }
- if (!fileSystem.isFile(pkgPath)) {
- throw new IOException("Package doesn't exist as a resource: " +
- pkg.getName());
- }
- log.info("Adding resource {}", pkg.getName());
- LocalResourceType type = LocalResourceType.FILE;
- if ("archive".equals(pkg.getType())) {
- type = LocalResourceType.ARCHIVE;
- }
- LocalResource packageResource = fileSystem.createAmResource(
- pkgPath, type);
- launcher.addLocalResource(APP_PACKAGES_DIR, packageResource);
- }
-
- String agentConf = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getOption(AGENT_CONF, "");
- if (SliderUtils.isSet(agentConf)) {
- LocalResource agentConfRes = fileSystem.createAmResource(fileSystem
- .getFileSystem().resolvePath(new Path(agentConf)),
- LocalResourceType.FILE);
- launcher.addLocalResource(AGENT_CONFIG_FILE, agentConfRes);
- }
-
- String agentVer = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getOption(AGENT_VERSION, null);
- if (agentVer != null) {
- LocalResource agentVerRes = fileSystem.createAmResource(
- fileSystem.getFileSystem().resolvePath(new Path(agentVer)),
- LocalResourceType.FILE);
- launcher.addLocalResource(AGENT_VERSION_FILE, agentVerRes);
- }
-
- if (SliderUtils.isHadoopClusterSecure(getConfig())) {
- providerUtils.localizeServiceKeytabs(launcher, instanceDefinition,
- fileSystem, getClusterName());
- }
-
- MapOperations amComponent = instanceDefinition.
- getAppConfOperations().getComponent(COMPONENT_AM);
- if (providerUtils.hasTwoWaySSLEnabled(amComponent)) {
- providerUtils.localizeContainerSSLResources(launcher, container,
- fileSystem, getClusterName());
- }
-
- if (providerUtils.areStoresRequested(appComponent)) {
- providerUtils.localizeContainerSecurityStores(launcher, container,
- roleName, fileSystem, instanceDefinition, appComponent,
- getClusterName());
- }
-
- //add the configuration resources
- launcher.addLocalResources(fileSystem.submitDirectory(
- generatedConfPath,
- PROPAGATED_CONF_DIR_NAME));
-
- if (appComponent.getOptionBool(AM_CONFIG_GENERATION, false)) {
- // build and localize configuration files
- Map> configurations =
- buildCommandConfigurations(instanceDefinition.getAppConfOperations(),
- instanceDefinition.getInternalOperations(),
- container.getId().toString(), roleName, roleGroup);
- for (ConfigFile configFile : getMetaInfo(roleGroup)
- .getComponentConfigFiles(roleGroup)) {
- localizeConfigFile(launcher, roleName, roleGroup, configFile,
- configurations, launcher.getEnv(), fileSystem);
- }
- }
-
- String label = getContainerLabel(container, roleName, roleGroup);
- CommandLineBuilder operation = new CommandLineBuilder();
-
- String pythonExec = instanceDefinition.getAppConfOperations()
- .getGlobalOptions().getOption(SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH,
- PYTHON_EXE);
-
- operation.add(pythonExec);
-
- operation.add(scriptPath);
- operation.add(ARG_LABEL, label);
- operation.add(ARG_ZOOKEEPER_QUORUM);
- operation.add(getClusterOptionPropertyValue(OptionKeys.ZOOKEEPER_QUORUM));
- operation.add(ARG_ZOOKEEPER_REGISTRY_PATH);
- operation.add(getZkRegistryPath());
-
- String debugCmd = agentLaunchParameter.getNextLaunchParameter(roleGroup);
- if (SliderUtils.isSet(debugCmd)) {
- operation.add(ARG_DEBUG);
- operation.add(debugCmd);
- }
-
- operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/"
- + AGENT_OUT_FILE + " 2>&1");
-
- launcher.addCommand(operation.build());
-
- // localize addon package
- String addonAppDefString = instanceDefinition.getAppConfOperations()
- .getGlobalOptions().getOption(ADDONS, null);
- log.debug("All addon appdefs: {}", addonAppDefString);
- if (addonAppDefString != null) {
- Scanner scanner = new Scanner(addonAppDefString).useDelimiter(",");
- while (scanner.hasNext()) {
- String addonAppDef = scanner.next();
- String addonAppDefPath = instanceDefinition
- .getAppConfOperations().getGlobalOptions().get(addonAppDef);
- log.debug("Addon package {} is stored at: {}", addonAppDef, addonAppDefPath);
- LocalResource addonPkgRes = fileSystem.createAmResource(
- fileSystem.getFileSystem().resolvePath(new Path(addonAppDefPath)),
- LocalResourceType.ARCHIVE);
- launcher.addLocalResource(ADDON_DEFINITION_DIR + "/" + addonAppDef, addonPkgRes);
- }
- log.debug("Metainfo map for master and addon: {}",
- packageMetainfo.toString());
- }
-
- // Additional files to localize in addition to the application def
- String appResourcesString = instanceDefinition.getAppConfOperations()
- .getGlobalOptions().getOption(APP_RESOURCES, null);
- log.info("Configuration value for extra resources to localize: {}", appResourcesString);
- if (null != appResourcesString) {
- try (Scanner scanner = new Scanner(appResourcesString).useDelimiter(",")) {
- while (scanner.hasNext()) {
- String resource = scanner.next();
- Path resourcePath = new Path(resource);
- LocalResource extraResource = fileSystem.createAmResource(
- fileSystem.getFileSystem().resolvePath(resourcePath),
- LocalResourceType.FILE);
- String destination = APP_RESOURCES_DIR + "/" + resourcePath.getName();
- log.info("Localizing {} to {}", resourcePath, destination);
- // TODO Can we try harder to avoid collisions?
- launcher.addLocalResource(destination, extraResource);
- }
- }
- }
-
- // initialize addon pkg states for all componentInstanceStatus
- Map pkgStatuses = new TreeMap<>();
- for (Metainfo appPkg : packageMetainfo.values()) {
- // check each component of that addon to see if they apply to this
- // component 'role'
- for (ComponentsInAddonPackage comp : appPkg.getApplicationPackage()
- .getComponents()) {
- log.debug("Current component: {} component in metainfo: {}", roleName,
- comp.getName());
- if (comp.getName().equals(roleGroup)
- || comp.getName().equals(ADDON_FOR_ALL_COMPONENTS)) {
- pkgStatuses.put(appPkg.getApplicationPackage().getName(), State.INIT);
- }
- }
- }
- log.debug("For component: {} pkg status map: {}", roleName,
- pkgStatuses.toString());
-
- // initialize the component instance state
- getComponentStatuses().put(label,
- new ComponentInstanceState(
- roleGroup,
- container.getId(),
- getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME),
- pkgStatuses));
- }
-
- @VisibleForTesting
- protected void localizeConfigFile(ContainerLauncher launcher,
- String roleName, String roleGroup,
- ConfigFile configFile,
- Map> configs,
- MapOperations env,
- SliderFileSystem fileSystem)
- throws IOException {
- ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType());
- providerUtils.localizeConfigFile(launcher, roleName, roleGroup,
- configFile.getDictionaryName(), configFormat, configFile.getFileName(),
- configs, env, fileSystem, getClusterName());
- }
-
- /**
- * build the zookeeper registry path.
- *
- * @return the path the service registered at
- * @throws NullPointerException if the service has not yet registered
- */
- private String getZkRegistryPath() {
- Preconditions.checkNotNull(yarnRegistry, "Yarn registry not bound");
- String path = yarnRegistry.getAbsoluteSelfRegistrationPath();
- Preconditions.checkNotNull(path, "Service record path not defined");
- return path;
- }
-
- @Override
- public void rebuildContainerDetails(List liveContainers,
- String applicationId, Map providerRoleMap) {
- for (Container container : liveContainers) {
- // get the role name and label
- ProviderRole role = providerRoleMap.get(ContainerPriority
- .extractRole(container));
- if (role != null) {
- String roleName = role.name;
- String roleGroup = role.group;
- String label = getContainerLabel(container, roleName, roleGroup);
- log.info("Rebuilding in-memory: container {} in role {} in cluster {}",
- container.getId(), roleName, applicationId);
- getComponentStatuses().put(label,
- new ComponentInstanceState(roleGroup, container.getId(),
- applicationId));
- } else {
- log.warn("Role not found for container {} in cluster {}",
- container.getId(), applicationId);
- }
- }
- }
-
- @Override
- public boolean isSupportedRole(String role) {
- return true;
- }
-
- /**
- * Handle registration calls from the agents
- *
- * @param registration registration entry
- *
- * @return response
- */
- @Override
- public RegistrationResponse handleRegistration(Register registration) {
- log.info("Handling registration: {}", registration);
- RegistrationResponse response = new RegistrationResponse();
- String label = registration.getLabel();
- String pkg = registration.getPkg();
- State agentState = registration.getActualState();
- String appVersion = registration.getAppVersion();
-
- log.info("label: {} pkg: {}", label, pkg);
-
- if (getComponentStatuses().containsKey(label)) {
- response.setResponseStatus(RegistrationStatus.OK);
- ComponentInstanceState componentStatus = getComponentStatuses().get(label);
- componentStatus.heartbeat(System.currentTimeMillis());
- updateComponentStatusWithAgentState(componentStatus, agentState);
-
- String roleName = getRoleName(label);
- String roleGroup = getRoleGroup(label);
- String containerId = getContainerId(label);
-
- if (SliderUtils.isSet(registration.getTags())) {
- tags.recordAssignedTag(roleName, containerId, registration.getTags());
- } else {
- response.setTags(tags.getTag(roleName, containerId));
- }
-
- String hostFqdn = registration.getPublicHostname();
- Map ports = registration.getAllocatedPorts();
- if (ports != null && !ports.isEmpty()) {
- processAllocatedPorts(hostFqdn, roleName, roleGroup, containerId, ports);
- }
-
- Map folders = registration.getLogFolders();
- if (folders != null && !folders.isEmpty()) {
- publishFolderPaths(folders, containerId, roleName, hostFqdn);
- }
-
- // Set app version if empty. It gets unset during upgrade - why?
- checkAndSetContainerAppVersion(containerId, appVersion);
- } else {
- response.setResponseStatus(RegistrationStatus.FAILED);
- response.setLog("Label not recognized.");
- log.warn("Received registration request from unknown label {}", label);
- }
- log.info("Registration response: {}", response);
- return response;
- }
-
- // Checks if app version is empty. Sets it to the version as reported by the
- // container during registration phase.
- private void checkAndSetContainerAppVersion(String containerId,
- String appVersion) {
- StateAccessForProviders amState = getAmState();
- try {
- RoleInstance role = amState.getOwnedContainer(containerId);
- if (role != null) {
- String currentAppVersion = role.appVersion;
- log.debug("Container = {}, app version current = {} new = {}",
- containerId, currentAppVersion, appVersion);
- if (currentAppVersion == null
- || currentAppVersion.equals(APP_VERSION_UNKNOWN)) {
- amState.getOwnedContainer(containerId).appVersion = appVersion;
- }
- }
- } catch (NoSuchNodeException e) {
- // ignore - there is nothing to do if we don't find a container
- log.warn("Owned container {} not found - {}", containerId, e);
- }
- }
-
- /**
- * Handle heartbeat response from agents
- *
- * @param heartBeat incoming heartbeat from Agent
- *
- * @return response to send back
- */
- @Override
- public HeartBeatResponse handleHeartBeat(HeartBeat heartBeat) {
- log.debug("Handling heartbeat: {}", heartBeat);
- HeartBeatResponse response = new HeartBeatResponse();
- long id = heartBeat.getResponseId();
- response.setResponseId(id + 1L);
-
- String label = heartBeat.getHostname();
- String pkg = heartBeat.getPackage();
-
- log.debug("package received: " + pkg);
-
- String roleName = getRoleName(label);
- String roleGroup = getRoleGroup(label);
- String containerId = getContainerId(label);
- boolean doUpgrade = false;
- if (isInUpgradeMode && upgradeContainers.contains(containerId)) {
- doUpgrade = true;
- }
-
- CommandScript cmdScript = getScriptPathForMasterPackage(roleGroup);
- List commands = getApplicationComponent(roleGroup).getCommands();
-
- if (!isDockerContainer(roleGroup) && !isYarnDockerContainer(roleGroup)
- && (cmdScript == null || cmdScript.getScript() == null)
- && commands.size() == 0) {
- log.error(
- "role.script is unavailable for {}. Commands will not be sent.",
- roleName);
- return response;
- }
-
- String scriptPath = null;
- long timeout = 600L;
- if (cmdScript != null) {
- scriptPath = cmdScript.getScript();
- timeout = cmdScript.getTimeout();
- }
-
- if (timeout == 0L) {
- timeout = 600L;
- }
-
- if (!getComponentStatuses().containsKey(label)) {
- // container is completed but still heart-beating, send terminate signal
- log.info(
- "Sending terminate signal to completed container (still heartbeating): {}",
- label);
- response.setTerminateAgent(true);
- return response;
- }
-
- List statuses = heartBeat.getComponentStatus();
- if (statuses != null && !statuses.isEmpty()) {
- log.info("status from agent: " + statuses.toString());
- for(ComponentStatus status : statuses){
- providerUtils.updateServiceRecord(getAmState(), yarnRegistry,
- containerId, roleName, Collections.singletonList(status.getIp()),
- status.getHostname());
- }
- }
-
- Boolean isMaster = isMaster(roleGroup);
- ComponentInstanceState componentStatus = getComponentStatuses().get(label);
- componentStatus.heartbeat(System.currentTimeMillis());
- if (doUpgrade) {
- switch (componentStatus.getState()) {
- case STARTED:
- componentStatus.setTargetState(State.UPGRADED);
- break;
- case UPGRADED:
- componentStatus.setTargetState(State.STOPPED);
- break;
- case STOPPED:
- componentStatus.setTargetState(State.TERMINATING);
- break;
- default:
- break;
- }
- log.info("Current state = {} target state {}",
- componentStatus.getState(), componentStatus.getTargetState());
- }
-
- if (appStopInitiated && !componentStatus.isStopInitiated()) {
- log.info("Stop initiated for label {}", label);
- componentStatus.setTargetState(State.STOPPED);
- componentStatus.setStopInitiated(true);
- }
-
- publishConfigAndExportGroups(heartBeat, componentStatus, roleGroup);
- CommandResult result = null;
- List reports = heartBeat.getReports();
- if (SliderUtils.isNotEmpty(reports)) {
- CommandReport report = reports.get(0);
- Map ports = report.getAllocatedPorts();
- if (SliderUtils.isNotEmpty(ports)) {
- processAllocatedPorts(heartBeat.getFqdn(), roleName, roleGroup, containerId, ports);
- }
- result = CommandResult.getCommandResult(report.getStatus());
- Command command = Command.getCommand(report.getRoleCommand());
- componentStatus.applyCommandResult(result, command, pkg);
- log.info("Component operation. Status: {}; new container state: {};"
- + " new component state: {}", result,
- componentStatus.getContainerState(), componentStatus.getState());
-
- if (command == Command.INSTALL && SliderUtils.isNotEmpty(report.getFolders())) {
- publishFolderPaths(report.getFolders(), containerId, roleName, heartBeat.getFqdn());
- }
- }
-
- int waitForCount = getAmState().getInstanceDefinitionSnapshot().
- getAppConfOperations().getComponentOptInt(roleGroup, WAIT_HEARTBEAT, 0);
-
- if (id < waitForCount) {
- log.info("Waiting until heartbeat count {}. Current val: {}", waitForCount, id);
- getComponentStatuses().put(label, componentStatus);
- return response;
- }
-
- Command command = componentStatus.getNextCommand(doUpgrade);
- try {
- if (Command.NOP != command) {
- log.debug("For comp {} pkg {} issuing {}", roleName,
- componentStatus.getNextPkgToInstall(), command.toString());
- if (command == Command.INSTALL) {
- log.info("Installing {} on {}.", roleName, containerId);
- if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)){
- addInstallDockerCommand(roleName, roleGroup, containerId,
- response, null, timeout);
- } else if (scriptPath != null) {
- addInstallCommand(roleName, roleGroup, containerId, response,
- scriptPath, null, timeout, null);
- } else {
- // commands
- ComponentCommand installCmd = null;
- for (ComponentCommand compCmd : commands) {
- if (compCmd.getName().equals("INSTALL")) {
- installCmd = compCmd;
- }
- }
- addInstallCommand(roleName, roleGroup, containerId, response, null,
- installCmd, timeout, null);
- }
- componentStatus.commandIssued(command);
- } else if (command == Command.INSTALL_ADDON) {
- String nextPkgToInstall = componentStatus.getNextPkgToInstall();
- // retrieve scriptPath or command of that package for the component
- for (ComponentsInAddonPackage comp : packageMetainfo
- .get(nextPkgToInstall).getApplicationPackage().getComponents()) {
- // given nextPkgToInstall and roleName is determined, the if below
- // should only execute once per heartbeat
- log.debug("Addon component: {} pkg: {} script: {}", comp.getName(),
- nextPkgToInstall, comp.getCommandScript().getScript());
- if (comp.getName().equals(roleGroup)
- || comp.getName().equals(ADDON_FOR_ALL_COMPONENTS)) {
- scriptPath = comp.getCommandScript().getScript();
- if (scriptPath != null) {
- addInstallCommand(roleName, roleGroup, containerId, response,
- scriptPath, null, timeout, nextPkgToInstall);
- } else {
- ComponentCommand installCmd = null;
- for (ComponentCommand compCmd : comp.getCommands()) {
- if (compCmd.getName().equals("INSTALL")) {
- installCmd = compCmd;
- }
- }
- addInstallCommand(roleName, roleGroup, containerId, response,
- null, installCmd, timeout, nextPkgToInstall);
- }
- }
- }
- componentStatus.commandIssued(command);
- } else if (command == Command.START) {
- // check against dependencies
- boolean canExecute = commandOrder.canExecute(roleGroup, command, getComponentStatuses().values());
- if (canExecute) {
- log.info("Starting {} on {}.", roleName, containerId);
- if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)){
- addStartDockerCommand(roleName, roleGroup, containerId,
- response, null, timeout, false);
- } else if (scriptPath != null) {
- addStartCommand(roleName,
- roleGroup,
- containerId,
- response,
- scriptPath,
- null,
- null,
- timeout,
- isMarkedAutoRestart(roleGroup));
- } else {
- ComponentCommand startCmd = null;
- for (ComponentCommand compCmd : commands) {
- if (compCmd.getName().equals("START")) {
- startCmd = compCmd;
- }
- }
- ComponentCommand stopCmd = null;
- for (ComponentCommand compCmd : commands) {
- if (compCmd.getName().equals("STOP")) {
- stopCmd = compCmd;
- }
- }
- addStartCommand(roleName, roleGroup, containerId, response, null,
- startCmd, stopCmd, timeout, false);
- }
- componentStatus.commandIssued(command);
- } else {
- log.info("Start of {} on {} delayed as dependencies have not started.", roleName, containerId);
- }
- } else if (command == Command.UPGRADE) {
- addUpgradeCommand(roleName, roleGroup, containerId, response,
- scriptPath, timeout);
- componentStatus.commandIssued(command, true);
- } else if (command == Command.STOP) {
- log.info("Stop command being sent to container with id {}",
- containerId);
- addStopCommand(roleName, roleGroup, containerId, response, scriptPath,
- timeout, doUpgrade);
- componentStatus.commandIssued(command);
- } else if (command == Command.TERMINATE) {
- log.info("A formal terminate command is being sent to container {}"
- + " in state {}", label, componentStatus.getState());
- response.setTerminateAgent(true);
- }
- }
-
- // if there is no outstanding command then retrieve config
- if (isMaster && componentStatus.getState() == State.STARTED
- && command == Command.NOP) {
- if (!componentStatus.getConfigReported()) {
- log.info("Requesting applied config for {} on {}.", roleName, containerId);
- if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)){
- addGetConfigDockerCommand(roleName, roleGroup, containerId, response);
- } else {
- addGetConfigCommand(roleName, roleGroup, containerId, response);
- }
- }
- }
-
- // if restart is required then signal
- response.setRestartEnabled(false);
- if (componentStatus.getState() == State.STARTED
- && command == Command.NOP && isMarkedAutoRestart(roleGroup)) {
- response.setRestartEnabled(true);
- }
-
- //If INSTALL_FAILED and no INSTALL is scheduled let the agent fail
- if (componentStatus.getState() == State.INSTALL_FAILED
- && command == Command.NOP) {
- log.warn("Sending terminate signal to container that failed installation: {}", label);
- response.setTerminateAgent(true);
- }
-
- } catch (SliderException e) {
- log.warn("Component instance failed operation.", e);
- componentStatus.applyCommandResult(CommandResult.FAILED, command, null);
- }
-
- log.debug("Heartbeat response: " + response);
- return response;
- }
-
- private boolean isDockerContainer(String roleGroup) {
- String type = getApplicationComponent(roleGroup).getType();
- if (SliderUtils.isSet(type)) {
- return type.toLowerCase().equals(SliderUtils.DOCKER) || type.toLowerCase().equals(SliderUtils.DOCKER_YARN);
- }
- return false;
- }
-
- private boolean isYarnDockerContainer(String roleGroup) {
- String type = getApplicationComponent(roleGroup).getType();
- if (SliderUtils.isSet(type)) {
- return type.toLowerCase().equals(SliderUtils.DOCKER_YARN);
- }
- return false;
- }
-
- protected void processAllocatedPorts(String fqdn,
- String roleName,
- String roleGroup,
- String containerId,
- Map ports) {
- RoleInstance instance;
- try {
- instance = getAmState().getOwnedContainer(containerId);
- } catch (NoSuchNodeException e) {
- log.warn("Failed to locate instance of container {}", containerId, e);
- instance = null;
- }
- for (Map.Entry port : ports.entrySet()) {
- String portname = port.getKey();
- String portNo = port.getValue();
- log.info("Recording allocated port for {} as {}", portname, portNo);
-
- // add the allocated ports to the global list as well as per container list
- // per container allocation will over-write each other in the global
- this.getAllocatedPorts().put(portname, portNo);
- this.getAllocatedPorts(containerId).put(portname, portNo);
- if (instance != null) {
- try {
- // if the returned value is not a single port number then there are no
- // meaningful way for Slider to use it during export
- // No need to error out as it may not be the responsibility of the component
- // to allocate port or the component may need an array of ports
- instance.registerPortEndpoint(Integer.valueOf(portNo), portname);
- } catch (NumberFormatException e) {
- log.warn("Failed to parse {}", portNo, e);
- }
- }
- }
-
- processAndPublishComponentSpecificData(ports, containerId, fqdn, roleGroup);
- processAndPublishComponentSpecificExports(ports, containerId, fqdn, roleName, roleGroup);
-
- // and update registration entries
- if (instance != null) {
- queueAccess.put(new RegisterComponentInstance(instance.getId(),
- roleName, roleGroup, 0, TimeUnit.MILLISECONDS));
- }
- }
-
- private void updateComponentStatusWithAgentState(
- ComponentInstanceState componentStatus, State agentState) {
- if (agentState != null) {
- componentStatus.setState(agentState);
- }
- }
-
- @Override
- public Map buildMonitorDetails(ClusterDescription clusterDesc) {
- Map details = super.buildMonitorDetails(clusterDesc);
- buildRoleHostDetails(details);
- return details;
- }
-
- public void applyInitialRegistryDefinitions(URL amWebURI,
- URL agentOpsURI,
- URL agentStatusURI,
- ServiceRecord serviceRecord)
- throws IOException {
- super.applyInitialRegistryDefinitions(amWebURI,
- serviceRecord);
-
- try {
- URL restURL = new URL(agentOpsURI, SLIDER_PATH_AGENTS);
- URL agentStatusURL = new URL(agentStatusURI, SLIDER_PATH_AGENTS);
-
- serviceRecord.addInternalEndpoint(
- new Endpoint(CustomRegistryConstants.AGENT_SECURE_REST_API,
- ProtocolTypes.PROTOCOL_REST,
- restURL.toURI()));
- serviceRecord.addInternalEndpoint(
- new Endpoint(CustomRegistryConstants.AGENT_ONEWAY_REST_API,
- ProtocolTypes.PROTOCOL_REST,
- agentStatusURL.toURI()));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
-
- // identify client component
- Component client = null;
- for (Component component : getMetaInfo().getApplication().getComponents()) {
- if (component.getCategory().equals("CLIENT")) {
- client = component;
- break;
- }
- }
- if (client == null) {
- log.info("No client component specified, not publishing client configs");
- return;
- }
-
- // register AM-generated client configs
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- MapOperations clientOperations = appConf.getOrAddComponent(client.getName());
- appConf.resolve();
- if (!clientOperations.getOptionBool(AM_CONFIG_GENERATION,
- false)) {
- log.info("AM config generation is false, not publishing client configs");
- return;
- }
-
- // build and localize configuration files
- Map> configurations = new TreeMap<>();
- Map tokens = providerUtils.getStandardTokenMap(appConf,
- getAmState().getInternalsSnapshot(), client.getName(),
- client.getName(), getClusterName());
-
- for (ConfigFile configFile : getMetaInfo().getComponentConfigFiles(client.getName())) {
- addNamedConfiguration(configFile.getDictionaryName(),
- appConf.getGlobalOptions().options, configurations, tokens, null,
- client.getName(), client.getName());
- if (appConf.getComponent(client.getName()) != null) {
- addNamedConfiguration(configFile.getDictionaryName(),
- appConf.getComponent(client.getName()).options, configurations,
- tokens, null, client.getName(), client.getName());
- }
- }
-
- //do a final replacement of re-used configs
- dereferenceAllConfigs(configurations);
-
- for (ConfigFile configFile : getMetaInfo().getComponentConfigFiles(client.getName())) {
- ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType());
-
- Map config = configurations.get(configFile.getDictionaryName());
- ConfigUtils.prepConfigForTemplateOutputter(configFormat, config,
- fileSystem, getClusterName(),
- new File(configFile.getFileName()).getName());
- PublishedConfiguration publishedConfiguration =
- new PublishedConfiguration(configFile.getDictionaryName(),
- config.entrySet());
- getAmState().getPublishedSliderConfigurations().put(
- configFile.getDictionaryName(), publishedConfiguration);
- log.info("Publishing AM configuration {}", configFile.getDictionaryName());
- }
- }
-
- @Override
- public void notifyContainerCompleted(ContainerId containerId) {
- // containers get allocated and free'ed without being assigned to any
- // component - so many of the data structures may not be initialized
- if (containerId != null) {
- String containerIdStr = containerId.toString();
- if (getComponentInstanceData().containsKey(containerIdStr)) {
- getComponentInstanceData().remove(containerIdStr);
- log.info("Removing container specific data for {}", containerIdStr);
- publishComponentInstanceData();
- }
-
- if (this.allocatedPorts.containsKey(containerIdStr)) {
- Map portsByContainerId = getAllocatedPorts(containerIdStr);
- this.allocatedPorts.remove(containerIdStr);
- // free up the allocations from global as well
- // if multiple containers allocate global ports then last one
- // wins and similarly first one removes it - its not supported anyway
- for(String portName : portsByContainerId.keySet()) {
- getAllocatedPorts().remove(portName);
- }
-
- }
-
- String componentName = null;
- synchronized (this.componentStatuses) {
- for (String label : getComponentStatuses().keySet()) {
- if (label.startsWith(containerIdStr)) {
- componentName = getRoleName(label);
- log.info("Removing component status for label {}", label);
- getComponentStatuses().remove(label);
- }
- }
- }
-
- tags.releaseTag(componentName, containerIdStr);
-
- synchronized (this.containerExportsMap) {
- Set containerExportSets = containerExportsMap.get(containerIdStr);
- if (containerExportSets != null) {
- for (String containerExportStr : containerExportSets) {
- String[] parts = containerExportStr.split(":");
- Map> exportGroup = getCurrentExports(parts[0]);
- List exports = exportGroup.get(parts[1]);
- List exportToRemove = new ArrayList();
- for (ExportEntry export : exports) {
- if (containerIdStr.equals(export.getContainerId())) {
- exportToRemove.add(export);
- }
- }
- exports.removeAll(exportToRemove);
- }
- log.info("Removing container exports for {}", containerIdStr);
- containerExportsMap.remove(containerIdStr);
- }
- }
- }
- }
-
- /**
- * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
- *
- * @param instanceDefinition
- */
- private void readAndSetHeartbeatMonitoringInterval(AggregateConf instanceDefinition) {
- String hbMonitorInterval = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getOption(HEARTBEAT_MONITOR_INTERVAL,
- Integer.toString(DEFAULT_HEARTBEAT_MONITOR_INTERVAL));
- try {
- setHeartbeatMonitorInterval(Integer.parseInt(hbMonitorInterval));
- } catch (NumberFormatException e) {
- log.warn(
- "Bad value {} for {}. Defaulting to ",
- hbMonitorInterval,
- HEARTBEAT_MONITOR_INTERVAL,
- DEFAULT_HEARTBEAT_MONITOR_INTERVAL);
- }
- }
-
- /**
- * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
- *
- * @param instanceDefinition
- */
- private void initializeAgentDebugCommands(AggregateConf instanceDefinition) {
- String launchParameterStr = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getOption(AGENT_INSTANCE_DEBUG_DATA, "");
- agentLaunchParameter = new AgentLaunchParameter(launchParameterStr);
- }
-
- @VisibleForTesting
- protected Map getLogFolderExports() {
- return logFolderExports;
- }
-
- @VisibleForTesting
- protected Map getWorkFolderExports() {
- return workFolderExports;
- }
-
- protected Metainfo getMetaInfo() {
- return getMetaInfo(null);
- }
-
- @VisibleForTesting
- protected Metainfo getMetaInfo(String roleGroup) {
- String mapKey = DEFAULT_METAINFO_MAP_KEY;
- if (roleGroup != null) {
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX,
- DEFAULT_METAINFO_MAP_KEY);
- }
- MetainfoHolder mh = this.metaInfoMap.get(mapKey);
- if (mh == null) {
- return null;
- }
- return mh.metaInfo;
- }
-
- @VisibleForTesting
- protected Map getComponentStatuses() {
- return componentStatuses;
- }
-
- @VisibleForTesting
- protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
- String appDef, boolean addonPackage) throws IOException,
- BadConfigException {
- return AgentUtils.getApplicationMetainfo(fileSystem, appDef, addonPackage);
- }
-
- @VisibleForTesting
- protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
- String appDef) throws IOException, BadConfigException {
- return getApplicationMetainfo(fileSystem, appDef, false);
- }
-
- @VisibleForTesting
- protected void setHeartbeatMonitorInterval(int heartbeatMonitorInterval) {
- this.heartbeatMonitorInterval = heartbeatMonitorInterval;
- }
-
- public void setInUpgradeMode(boolean inUpgradeMode) {
- this.isInUpgradeMode = inUpgradeMode;
- }
-
- public void addUpgradeContainers(Set upgradeContainers) {
- this.upgradeContainers.addAll(upgradeContainers);
- }
-
- public void setAppStopInitiated(boolean appStopInitiated) {
- this.appStopInitiated = appStopInitiated;
- }
-
- /**
- * Read all default configs
- *
- * @param fileSystem fs
- * @param appDef app default path
- * @param metainfo metadata
- *
- * @return configuration maps
- *
- * @throws IOException
- */
- protected Map initializeDefaultConfigs(SliderFileSystem fileSystem,
- String appDef, Metainfo metainfo) throws IOException {
- Map defaultConfigMap = new HashMap<>();
- if (SliderUtils.isNotEmpty(metainfo.getApplication().getConfigFiles())) {
- for (ConfigFile configFile : metainfo.getApplication().getConfigFiles()) {
- DefaultConfig config = null;
- try {
- config = AgentUtils.getDefaultConfig(fileSystem, appDef, configFile.getDictionaryName() + ".xml");
- } catch (IOException e) {
- log.warn("Default config file not found. Only the config as input during create will be applied for {}",
- configFile.getDictionaryName());
- }
- if (config != null) {
- defaultConfigMap.put(configFile.getDictionaryName(), config);
- }
- }
- }
-
- return defaultConfigMap;
- }
-
- protected Map getDefaultConfigs(String roleGroup) {
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- String mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX,
- DEFAULT_METAINFO_MAP_KEY);
- return metaInfoMap.get(mapKey).defaultConfigs;
- }
-
- private int getHeartbeatMonitorInterval() {
- return this.heartbeatMonitorInterval;
- }
-
- private String getClusterName() {
- if (SliderUtils.isUnset(clusterName)) {
- clusterName = getAmState().getInternalsSnapshot().get(OptionKeys.APPLICATION_NAME);
- }
- return clusterName;
- }
-
- @VisibleForTesting
- protected void publishApplicationInstanceData(String name, String description,
- Iterable> entries) {
- providerUtils.publishApplicationInstanceData(name, description, entries,
- getAmState());
- }
-
- /**
- * Get a list of all hosts for all role/container per role
- *
- * @return the map of role->node
- */
- protected Map> getRoleClusterNodeMapping() {
- return amState.getRoleClusterNodeMapping();
- }
-
- private String getContainerLabel(Container container, String role, String group) {
- if (role.equals(group)) {
- return container.getId().toString() + LABEL_MAKER + role;
- } else {
- return container.getId().toString() + LABEL_MAKER + role + LABEL_MAKER +
- group;
- }
- }
-
- protected String getClusterInfoPropertyValue(String name) {
- StateAccessForProviders accessor = getAmState();
- assert accessor.isApplicationLive();
- ClusterDescription description = accessor.getClusterStatus();
- return description.getInfo(name);
- }
-
- protected String getClusterOptionPropertyValue(String name)
- throws BadConfigException {
- StateAccessForProviders accessor = getAmState();
- assert accessor.isApplicationLive();
- ClusterDescription description = accessor.getClusterStatus();
- return description.getMandatoryOption(name);
- }
-
- /**
- * Lost heartbeat from the container - release it and ask for a replacement (async operation)
- *
- * @param label
- * @param containerId
- */
- protected void lostContainer(
- String label,
- ContainerId containerId) {
- getComponentStatuses().remove(label);
- getQueueAccess().put(new ProviderReportedContainerLoss(containerId));
- }
-
- /**
- * Build the provider status, can be empty
- *
- * @return the provider status - map of entries to add to the info section
- */
- public Map buildProviderStatus() {
- Map stats = new HashMap();
- return stats;
- }
-
- @VisibleForTesting
- protected void publishFolderPaths(
- Map folders, String containerId, String componentName, String hostFqdn) {
- providerUtils.publishFolderPaths(folders, containerId, componentName, hostFqdn,
- getAmState(), getLogFolderExports(), getWorkFolderExports());
- }
-
- /**
- * Process return status for component instances
- *
- * @param heartBeat
- * @param componentStatus
- */
- protected void publishConfigAndExportGroups(HeartBeat heartBeat,
- ComponentInstanceState componentStatus, String componentGroup) {
- List statuses = heartBeat.getComponentStatus();
- if (statuses != null && !statuses.isEmpty()) {
- log.info("Processing {} status reports.", statuses.size());
- for (ComponentStatus status : statuses) {
- log.info("Status report: {}", status.toString());
-
- if (status.getConfigs() != null) {
- Application application = getMetaInfo(componentGroup).getApplication();
-
- if ((!canAnyMasterPublishConfig(componentGroup) || canPublishConfig(componentGroup)) &&
- !getAmState().getAppConfSnapshot().getComponentOptBool(
- componentGroup, AM_CONFIG_GENERATION, false)) {
- // If no Master can explicitly publish then publish if its a master
- // Otherwise, wait till the master that can publish is ready
-
- Set exportedConfigs = new HashSet();
- String exportedConfigsStr = application.getExportedConfigs();
- boolean exportedAllConfigs = exportedConfigsStr == null || exportedConfigsStr.isEmpty();
- if (!exportedAllConfigs) {
- for (String exportedConfig : exportedConfigsStr.split(",")) {
- if (exportedConfig.trim().length() > 0) {
- exportedConfigs.add(exportedConfig.trim());
- }
- }
- }
-
- for (String key : status.getConfigs().keySet()) {
- if ((!exportedAllConfigs && exportedConfigs.contains(key)) ||
- exportedAllConfigs) {
- Map configs = status.getConfigs().get(key);
- publishApplicationInstanceData(key, key, configs.entrySet());
- }
- }
- }
-
- List appExportGroups = application.getExportGroups();
- boolean hasExportGroups = SliderUtils.isNotEmpty(appExportGroups);
-
- Set appExports = new HashSet();
- String appExportsStr = getApplicationComponent(componentGroup).getAppExports();
- if (SliderUtils.isSet(appExportsStr)) {
- for (String appExport : appExportsStr.split(",")) {
- if (!appExport.trim().isEmpty()) {
- appExports.add(appExport.trim());
- }
- }
- }
-
- if (hasExportGroups && !appExports.isEmpty()) {
- String configKeyFormat = "${site.%s.%s}";
- String hostKeyFormat = "${%s_HOST}";
-
- // publish export groups if any
- Map replaceTokens = new HashMap();
- for (Map.Entry> entry : getRoleClusterNodeMapping().entrySet()) {
- String hostName = providerUtils.getHostsList(
- entry.getValue().values(), true).iterator().next();
- replaceTokens.put(String.format(hostKeyFormat, entry.getKey().toUpperCase(Locale.ENGLISH)), hostName);
- }
-
- for (String key : status.getConfigs().keySet()) {
- Map configs = status.getConfigs().get(key);
- for (String configKey : configs.keySet()) {
- String lookupKey = String.format(configKeyFormat, key, configKey);
- replaceTokens.put(lookupKey, configs.get(configKey));
- }
- }
-
- Set modifiedGroups = new HashSet();
- for (ExportGroup exportGroup : appExportGroups) {
- List exports = exportGroup.getExports();
- if (SliderUtils.isNotEmpty(exports)) {
- String exportGroupName = exportGroup.getName();
- ConcurrentHashMap> map =
- (ConcurrentHashMap>)getCurrentExports(exportGroupName);
- for (Export export : exports) {
- if (canBeExported(exportGroupName, export.getName(), appExports)) {
- String value = export.getValue();
- // replace host names
- for (String token : replaceTokens.keySet()) {
- if (value.contains(token)) {
- value = value.replace(token, replaceTokens.get(token));
- }
- }
- ExportEntry entry = new ExportEntry();
- entry.setLevel(APPLICATION_TAG);
- entry.setValue(value);
- entry.setUpdatedTime(new Date().toString());
- // over-write, app exports are singletons
- map.put(export.getName(), new ArrayList(Arrays.asList(entry)));
- log.info("Preparing to publish. Key {} and Value {}", export.getName(), value);
- }
- }
- modifiedGroups.add(exportGroupName);
- }
- }
- publishModifiedExportGroups(modifiedGroups);
- }
-
- log.info("Received and processed config for {}", heartBeat.getHostname());
- componentStatus.setConfigReported(true);
-
- }
- }
- }
- }
-
- private boolean canBeExported(String exportGroupName, String name, Set appExports) {
- return appExports.contains(String.format("%s-%s", exportGroupName, name));
- }
-
- protected Map> getCurrentExports(String groupName) {
- if (!this.exportGroups.containsKey(groupName)) {
- synchronized (this.exportGroups) {
- if (!this.exportGroups.containsKey(groupName)) {
- this.exportGroups.put(groupName, new ConcurrentHashMap>());
- }
- }
- }
-
- return this.exportGroups.get(groupName);
- }
-
- private void publishModifiedExportGroups(Set modifiedGroups) {
- for (String roleGroup : modifiedGroups) {
- Map> entries = this.exportGroups.get(roleGroup);
- // Publish in old format for the time being
- Map simpleEntries = new HashMap();
- for (Entry> entry : entries.entrySet()) {
- List exports = entry.getValue();
- if (SliderUtils.isNotEmpty(exports)) {
- // there is no support for multiple exports per name - so extract only the first one
- simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue());
- }
- }
- publishApplicationInstanceData(roleGroup, roleGroup,
- simpleEntries.entrySet());
-
- PublishedExports exports = new PublishedExports(roleGroup);
- exports.setUpdated(new Date().getTime());
- exports.putValues(entries.entrySet());
- getAmState().getPublishedExportsSet().put(roleGroup, exports);
- }
- }
-
- /** Publish component instance specific data if the component demands it */
- protected void processAndPublishComponentSpecificData(Map ports,
- String containerId,
- String hostFqdn,
- String componentGroup) {
- String portVarFormat = "${site.%s}";
- String hostNamePattern = "${THIS_HOST}";
- Map toPublish = new HashMap();
-
- Application application = getMetaInfo(componentGroup).getApplication();
- for (Component component : application.getComponents()) {
- if (component.getName().equals(componentGroup)) {
- if (component.getComponentExports().size() > 0) {
-
- for (ComponentExport export : component.getComponentExports()) {
- String templateToExport = export.getValue();
- for (String portName : ports.keySet()) {
- boolean publishData = false;
- String portValPattern = String.format(portVarFormat, portName);
- if (templateToExport.contains(portValPattern)) {
- templateToExport = templateToExport.replace(portValPattern, ports.get(portName));
- publishData = true;
- }
- if (templateToExport.contains(hostNamePattern)) {
- templateToExport = templateToExport.replace(hostNamePattern, hostFqdn);
- publishData = true;
- }
- if (publishData) {
- toPublish.put(export.getName(), templateToExport);
- log.info("Publishing {} for name {} and container {}",
- templateToExport, export.getName(), containerId);
- }
- }
- }
- }
- }
- }
-
- if (toPublish.size() > 0) {
- Map perContainerData = null;
- if (!getComponentInstanceData().containsKey(containerId)) {
- perContainerData = new ConcurrentHashMap();
- } else {
- perContainerData = getComponentInstanceData().get(containerId);
- }
- perContainerData.putAll(toPublish);
- getComponentInstanceData().put(containerId, perContainerData);
- publishComponentInstanceData();
- }
- }
-
- /** Publish component instance specific data if the component demands it */
- protected void processAndPublishComponentSpecificExports(Map ports,
- String containerId,
- String hostFqdn,
- String compName,
- String compGroup) {
- String portVarFormat = "${site.%s}";
- String hostNamePattern = "${" + compGroup + "_HOST}";
-
- List appExportGroups = getMetaInfo(compGroup).getApplication().getExportGroups();
- Component component = getApplicationComponent(compGroup);
- if (component != null && SliderUtils.isSet(component.getCompExports())
- && SliderUtils.isNotEmpty(appExportGroups)) {
-
- Set compExports = new HashSet();
- String compExportsStr = component.getCompExports();
- for (String compExport : compExportsStr.split(",")) {
- if (!compExport.trim().isEmpty()) {
- compExports.add(compExport.trim());
- }
- }
-
- Date now = new Date();
- Set modifiedGroups = new HashSet();
- for (ExportGroup exportGroup : appExportGroups) {
- List exports = exportGroup.getExports();
- if (SliderUtils.isNotEmpty(exports)) {
- String exportGroupName = exportGroup.getName();
- ConcurrentHashMap> map =
- (ConcurrentHashMap>) getCurrentExports(exportGroupName);
- for (Export export : exports) {
- if (canBeExported(exportGroupName, export.getName(), compExports)) {
- log.info("Attempting to publish {} of group {} for component type {}",
- export.getName(), exportGroupName, compName);
- String templateToExport = export.getValue();
- for (String portName : ports.keySet()) {
- boolean publishData = false;
- String portValPattern = String.format(portVarFormat, portName);
- if (templateToExport.contains(portValPattern)) {
- templateToExport = templateToExport.replace(portValPattern, ports.get(portName));
- publishData = true;
- }
- if (templateToExport.contains(hostNamePattern)) {
- templateToExport = templateToExport.replace(hostNamePattern, hostFqdn);
- publishData = true;
- }
- if (publishData) {
- ExportEntry entryToAdd = new ExportEntry();
- entryToAdd.setLevel(COMPONENT_TAG);
- entryToAdd.setValue(templateToExport);
- entryToAdd.setUpdatedTime(now.toString());
- entryToAdd.setContainerId(containerId);
- entryToAdd.setTag(tags.getTag(compName, containerId));
-
- List existingList =
- map.putIfAbsent(export.getName(), new CopyOnWriteArrayList(Arrays.asList(entryToAdd)));
-
- // in-place edit, no lock needed
- if (existingList != null) {
- boolean updatedInPlace = false;
- for (ExportEntry entry : existingList) {
- if (containerId.toLowerCase(Locale.ENGLISH)
- .equals(entry.getContainerId())) {
- entryToAdd.setValue(templateToExport);
- entryToAdd.setUpdatedTime(now.toString());
- updatedInPlace = true;
- }
- }
- if (!updatedInPlace) {
- existingList.add(entryToAdd);
- }
- }
-
- log.info("Publishing {} for name {} and container {}",
- templateToExport, export.getName(), containerId);
- modifiedGroups.add(exportGroupName);
- synchronized (containerExportsMap) {
- if (!containerExportsMap.containsKey(containerId)) {
- containerExportsMap.put(containerId, new HashSet());
- }
- Set containerExportMaps = containerExportsMap.get(containerId);
- containerExportMaps.add(String.format("%s:%s", exportGroupName, export.getName()));
- }
- }
- }
- }
- }
- }
- }
- publishModifiedExportGroups(modifiedGroups);
- }
- }
-
- private void publishComponentInstanceData() {
- Map dataToPublish = new HashMap();
- for (String container : getComponentInstanceData().keySet()) {
- for (String prop : getComponentInstanceData().get(container).keySet()) {
- dataToPublish.put(
- container + "." + prop, getComponentInstanceData().get(container).get(prop));
- }
- }
- publishApplicationInstanceData(COMPONENT_DATA_TAG, COMPONENT_DATA_TAG, dataToPublish.entrySet());
- }
-
- /**
- * Return Component based on group
- *
- * @param roleGroup component group
- *
- * @return the component entry or null for no match
- */
- protected Component getApplicationComponent(String roleGroup) {
- Metainfo metainfo = getMetaInfo(roleGroup);
- if (metainfo == null) {
- return null;
- }
- return metainfo.getApplicationComponent(roleGroup);
- }
-
- /**
- * Extract script path from the application metainfo
- *
- * @param roleGroup component group
- * @return the script path or null for no match
- */
- protected CommandScript getScriptPathForMasterPackage(String roleGroup) {
- Component component = getApplicationComponent(roleGroup);
- if (component != null) {
- return component.getCommandScript();
- }
- return null;
- }
-
- /**
- * Is the role of type MASTER
- *
- * @param roleGroup component group
- *
- * @return true if the role category is MASTER
- */
- protected boolean isMaster(String roleGroup) {
- Component component = getApplicationComponent(roleGroup);
- if (component != null) {
- if (component.getCategory().equals("MASTER")) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * Can the role publish configuration
- *
- * @param roleGroup component group
- *
- * @return true if it can be pubished
- */
- protected boolean canPublishConfig(String roleGroup) {
- Component component = getApplicationComponent(roleGroup);
- if (component != null) {
- return Boolean.TRUE.toString().equals(component.getPublishConfig());
- }
- return false;
- }
-
- /**
- * Checks if the role is marked auto-restart
- *
- * @param roleGroup component group
- *
- * @return true if it is auto-restart
- */
- protected boolean isMarkedAutoRestart(String roleGroup) {
- Component component = getApplicationComponent(roleGroup);
- if (component != null) {
- return component.getAutoStartOnFailureBoolean();
- }
- return false;
- }
-
- /**
- * Can any master publish config explicitly, if not a random master is used
- *
- * @return true if the condition holds
- */
- protected boolean canAnyMasterPublishConfig(String roleGroup) {
- if (canAnyMasterPublish == null) {
- Application application = getMetaInfo(roleGroup).getApplication();
- if (application == null) {
- log.error("Malformed app definition: Expect application as root element in the metainfo.xml");
- } else {
- for (Component component : application.getComponents()) {
- if (Boolean.TRUE.toString().equals(component.getPublishConfig()) &&
- component.getCategory().equals("MASTER")) {
- canAnyMasterPublish = true;
- }
- }
- }
- }
-
- if (canAnyMasterPublish == null) {
- canAnyMasterPublish = false;
- }
- return canAnyMasterPublish;
- }
-
- private String getRoleName(String label) {
- int index1 = label.indexOf(LABEL_MAKER);
- int index2 = label.lastIndexOf(LABEL_MAKER);
- if (index1 == index2) {
- return label.substring(index1 + LABEL_MAKER.length());
- } else {
- return label.substring(index1 + LABEL_MAKER.length(), index2);
- }
- }
-
- private String getRoleGroup(String label) {
- return label.substring(label.lastIndexOf(LABEL_MAKER) + LABEL_MAKER.length());
- }
-
- private String getContainerId(String label) {
- return label.substring(0, label.indexOf(LABEL_MAKER));
- }
-
- /**
- * Add install command to the heartbeat response
- *
- * @param roleName
- * @param roleGroup
- * @param containerId
- * @param response
- * @param scriptPath
- * @param pkg
- * when this field is null, it indicates the command is for the
- * master package; while not null, for the package named by this
- * field
- * @throws SliderException
- */
- @VisibleForTesting
- protected void addInstallCommand(String roleName,
- String roleGroup,
- String containerId,
- HeartBeatResponse response,
- String scriptPath,
- ComponentCommand compCmd,
- long timeout,
- String pkg)
- throws SliderException {
- assert getAmState().isApplicationLive();
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-
- ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND);
- prepareExecutionCommand(cmd);
- String clusterName = getClusterName();
- cmd.setClusterName(clusterName);
- cmd.setRoleCommand(Command.INSTALL.toString());
- cmd.setServiceName(clusterName);
- cmd.setComponentName(roleName);
- cmd.setRole(roleName);
- cmd.setPkg(pkg);
- Map hostLevelParams = new TreeMap();
- hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
- hostLevelParams.put(PACKAGE_LIST, getPackageList(roleGroup));
- hostLevelParams.put(CONTAINER_ID, containerId);
- cmd.setHostLevelParams(hostLevelParams);
-
- Map> configurations =
- buildCommandConfigurations(appConf, getAmState().getInternalsSnapshot(),
- containerId, roleName, roleGroup);
- cmd.setConfigurations(configurations);
- Map> componentConfigurations = buildComponentConfigurations(appConf);
- cmd.setComponentConfigurations(componentConfigurations);
-
- if (SliderUtils.isSet(scriptPath)) {
- cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false));
- } else {
- // assume it to be default shell command
- ComponentCommand effectiveCommand = compCmd;
- if (effectiveCommand == null) {
- effectiveCommand = ComponentCommand.getDefaultComponentCommand("INSTALL");
- }
- cmd.setCommandParams(commandParametersSet(effectiveCommand, timeout, false));
- configurations.get("global").put("exec_cmd", effectiveCommand.getExec());
- }
-
- cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME));
-
- response.addExecutionCommand(cmd);
-
- log.debug("command looks like: {} ", cmd);
- }
-
- @VisibleForTesting
- protected void addInstallDockerCommand(String roleName,
- String roleGroup,
- String containerId,
- HeartBeatResponse response,
- ComponentCommand compCmd,
- long timeout)
- throws SliderException {
- assert getAmState().isApplicationLive();
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-
- ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND);
- prepareExecutionCommand(cmd);
- String clusterName = getClusterName();
- cmd.setClusterName(clusterName);
- cmd.setRoleCommand(Command.INSTALL.toString());
- cmd.setServiceName(clusterName);
- cmd.setComponentName(roleName);
- cmd.setRole(roleName);
- Map hostLevelParams = new TreeMap();
- hostLevelParams.put(PACKAGE_LIST, getPackageList(roleGroup));
- hostLevelParams.put(CONTAINER_ID, containerId);
- cmd.setHostLevelParams(hostLevelParams);
-
- Map> configurations = buildCommandConfigurations(
- appConf, getAmState().getInternalsSnapshot(), containerId, roleName,
- roleGroup);
- cmd.setConfigurations(configurations);
- Map> componentConfigurations = buildComponentConfigurations(appConf);
- cmd.setComponentConfigurations(componentConfigurations);
-
- ComponentCommand effectiveCommand = compCmd;
- if (compCmd == null) {
- effectiveCommand = new ComponentCommand();
- effectiveCommand.setName("INSTALL");
- effectiveCommand.setExec("DEFAULT");
- }
- cmd.setCommandParams(setCommandParameters(effectiveCommand, timeout, false));
- configurations.get("global").put("exec_cmd", effectiveCommand.getExec());
-
- cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME));
- cmd.addContainerDetails(roleGroup, getMetaInfo(roleGroup));
-
- Map dockerConfig = new HashMap();
- if(isYarnDockerContainer(roleGroup)){
- //put nothing
- cmd.setYarnDockerMode(true);
- } else {
- dockerConfig.put(
- "docker.command_path",
- getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
- "commandPath"));
- dockerConfig.put("docker.image_name",
- getConfigFromMetaInfo(roleGroup, "image"));
- }
- configurations.put("docker", dockerConfig);
-
- log.debug("Docker- command: {}", cmd.toString());
-
- response.addExecutionCommand(cmd);
- }
-
- private Map setCommandParameters(String scriptPath,
- long timeout, boolean recordConfig) {
- Map cmdParams = new TreeMap();
- cmdParams.put("service_package_folder",
- "${AGENT_WORK_ROOT}/work/app/definition/package");
- cmdParams.put("script", scriptPath);
- cmdParams.put("schema_version", "2.0");
- cmdParams.put("command_timeout", Long.toString(timeout));
- cmdParams.put("script_type", AbstractComponent.TYPE_PYTHON);
- cmdParams.put("record_config", Boolean.toString(recordConfig));
- return cmdParams;
- }
-
- private Map setCommandParameters(ComponentCommand compCmd,
- long timeout, boolean recordConfig) {
- Map cmdParams = new TreeMap();
- cmdParams.put("service_package_folder",
- "${AGENT_WORK_ROOT}/work/app/definition/package");
- cmdParams.put("command", compCmd.getExec());
- cmdParams.put("schema_version", "2.0");
- cmdParams.put("command_timeout", Long.toString(timeout));
- cmdParams.put("script_type", compCmd.getType());
- cmdParams.put("record_config", Boolean.toString(recordConfig));
- return cmdParams;
- }
-
- private Map> buildComponentConfigurations(
- ConfTreeOperations appConf) {
- return appConf.getComponents();
- }
-
- protected static String getPackageListFromApplication(Application application) {
- String pkgFormatString = "{\"type\":\"%s\",\"name\":\"%s\"}";
- String pkgListFormatString = "[%s]";
- List packages = new ArrayList<>();
- if (application != null) {
- if (application.getPackages().size() > 0) {
- // no-op if there are packages that are not OS-specific, as these
- // will be localized by AM rather than the Agent
- // this should be backwards compatible, as there was previously an
- // XML parsing bug that ensured non-OS-specific packages did not exist
- } else {
- List osSpecifics = application.getOSSpecifics();
- if (osSpecifics != null && osSpecifics.size() > 0) {
- for (OSSpecific osSpecific : osSpecifics) {
- if (osSpecific.getOsType().equals("any")) {
- for (OSPackage osPackage : osSpecific.getPackages()) {
- packages.add(String.format(pkgFormatString, osPackage.getType(), osPackage.getName()));
- }
- }
- }
- }
- }
- }
-
- if (!packages.isEmpty()) {
- return "[" + SliderUtils.join(packages, ",", false) + "]";
- } else {
- return "[]";
- }
- }
-
- private String getPackageList(String roleGroup) {
- return getPackageListFromApplication(getMetaInfo(roleGroup).getApplication());
- }
-
- private void prepareExecutionCommand(ExecutionCommand cmd) {
- cmd.setTaskId(taskId.incrementAndGet());
- cmd.setCommandId(cmd.getTaskId() + "-1");
- }
-
- private Map commandParametersSet(String scriptPath, long timeout, boolean recordConfig) {
- Map cmdParams = new TreeMap<>();
- cmdParams.put("service_package_folder",
- "${AGENT_WORK_ROOT}/work/app/definition/package");
- cmdParams.put("script", scriptPath);
- cmdParams.put("schema_version", "2.0");
- cmdParams.put("command_timeout", Long.toString(timeout));
- cmdParams.put("script_type", "PYTHON");
- cmdParams.put("record_config", Boolean.toString(recordConfig));
- return cmdParams;
- }
-
- private Map commandParametersSet(ComponentCommand compCmd, long timeout, boolean recordConfig) {
- Map cmdParams = new TreeMap<>();
- cmdParams.put("service_package_folder",
- "${AGENT_WORK_ROOT}/work/app/definition/package");
- cmdParams.put("command", compCmd.getExec());
- cmdParams.put("schema_version", "2.0");
- cmdParams.put("command_timeout", Long.toString(timeout));
- cmdParams.put("script_type", compCmd.getType());
- cmdParams.put("record_config", Boolean.toString(recordConfig));
- return cmdParams;
- }
-
- @VisibleForTesting
- protected void addStatusCommand(String roleName,
- String roleGroup,
- String containerId,
- HeartBeatResponse response,
- String scriptPath,
- long timeout)
- throws SliderException {
- assert getAmState().isApplicationLive();
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)) {
- addStatusDockerCommand(roleName, roleGroup, containerId, response,
- scriptPath, timeout);
- return;
- }
-
- StatusCommand cmd = new StatusCommand();
- String clusterName = getClusterName();
-
- cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
- cmd.setComponentName(roleName);
- cmd.setServiceName(clusterName);
- cmd.setClusterName(clusterName);
- cmd.setRoleCommand(StatusCommand.STATUS_COMMAND);
-
- Map hostLevelParams = new TreeMap();
- hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
- hostLevelParams.put(CONTAINER_ID, containerId);
- cmd.setHostLevelParams(hostLevelParams);
-
- cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false));
-
- Map> configurations =
- buildCommandConfigurations(appConf, getAmState().getInternalsSnapshot(),
- containerId, roleName, roleGroup);
-
- cmd.setConfigurations(configurations);
-
- response.addStatusCommand(cmd);
- }
-
- @VisibleForTesting
- protected void addStatusDockerCommand(String roleName,
- String roleGroup,
- String containerId,
- HeartBeatResponse response,
- String scriptPath,
- long timeout)
- throws SliderException {
- assert getAmState().isApplicationLive();
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-
- StatusCommand cmd = new StatusCommand();
- String clusterName = getClusterName();
-
- cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
- cmd.setComponentName(roleName);
- cmd.setServiceName(clusterName);
- cmd.setClusterName(clusterName);
- cmd.setRoleCommand(StatusCommand.STATUS_COMMAND);
-
- Map hostLevelParams = new TreeMap();
- hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
- hostLevelParams.put(CONTAINER_ID, containerId);
- cmd.setHostLevelParams(hostLevelParams);
-
- cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
-
- Map> configurations = buildCommandConfigurations(
- appConf, getAmState().getInternalsSnapshot(), containerId, roleName,
- roleGroup);
- Map dockerConfig = new HashMap();
- String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, "statusCommand");
- if (statusCommand == null) {
- if(isYarnDockerContainer(roleGroup)){
- //should complain the required field is null
- cmd.setYarnDockerMode(true);
- } else {
- statusCommand = "docker top "
- + containerId
- + " | grep \"\"";// default value
- }
- }
- dockerConfig.put("docker.status_command",statusCommand);
- configurations.put("docker", dockerConfig);
- cmd.setConfigurations(configurations);
- log.debug("Docker- status {}", cmd);
- response.addStatusCommand(cmd);
- }
-
- @VisibleForTesting
- protected void addGetConfigDockerCommand(String roleName, String roleGroup,
- String containerId, HeartBeatResponse response) throws SliderException {
- assert getAmState().isApplicationLive();
-
- StatusCommand cmd = new StatusCommand();
- String clusterName = getClusterName();
-
- cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
- cmd.setComponentName(roleName);
- cmd.setServiceName(clusterName);
- cmd.setClusterName(clusterName);
- cmd.setRoleCommand(StatusCommand.GET_CONFIG_COMMAND);
- Map hostLevelParams = new TreeMap();
- hostLevelParams.put(CONTAINER_ID, containerId);
- cmd.setHostLevelParams(hostLevelParams);
-
- hostLevelParams.put(CONTAINER_ID, containerId);
-
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- Map> configurations = buildCommandConfigurations(
- appConf, getAmState().getInternalsSnapshot(), containerId, roleName,
- roleGroup);
- Map dockerConfig = new HashMap();
- String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, "statusCommand");
- if (statusCommand == null) {
- if(isYarnDockerContainer(roleGroup)){
- //should complain the required field is null
- cmd.setYarnDockerMode(true);
- } else {
- statusCommand = "docker top "
- + containerId
- + " | grep \"\"";// default value
- }
- }
- dockerConfig.put("docker.status_command",statusCommand);
- configurations.put("docker", dockerConfig);
-
- cmd.setConfigurations(configurations);
- log.debug("Docker- getconfig command {}", cmd);
-
- response.addStatusCommand(cmd);
- }
-
- private String getConfigFromMetaInfoWithAppConfigOverriding(String roleGroup,
- String configName){
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- String containerName = getApplicationComponent(roleGroup)
- .getDockerContainers().get(0).getName();
- String composedConfigName = null;
- String appConfigValue = null;
- //if the configName is about port , mount, inputfile, then check differently
- if (configName.equals("containerPort") || configName.equals("hostPort")){
- composedConfigName = containerName + ".ports." + configName;
- } else
- if (configName.equals("containerMount")
- || configName.equals("hostMount")){
- composedConfigName = containerName + ".mounts." + configName;
- } else
- if (configName.equals("containerPath")
- || configName.equals("fileLocalPath")) {
- composedConfigName = containerName + ".inputFiles." + configName;
- } else {
- composedConfigName = containerName + "." + configName;
- }
- appConfigValue = appConf.getComponentOpt(roleGroup, composedConfigName,
- null);
- log.debug(
- "Docker- value from appconfig component: {} configName: {} value: {}",
- roleGroup, composedConfigName, appConfigValue);
- if (appConfigValue == null) {
- appConfigValue = getConfigFromMetaInfo(roleGroup, configName);
- log.debug(
- "Docker- value from metainfo component: {} configName: {} value: {}",
- roleGroup, configName, appConfigValue);
-
- }
- return appConfigValue;
- }
-
- @VisibleForTesting
- protected void addStartDockerCommand(String roleName, String roleGroup,
- String containerId, HeartBeatResponse response,
- ComponentCommand startCommand, long timeout, boolean isMarkedAutoRestart)
- throws
- SliderException {
- assert getAmState().isApplicationLive();
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
-
- ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND);
- prepareExecutionCommand(cmd);
- String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
- String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME);
- cmd.setHostname(hostName);
- cmd.setClusterName(clusterName);
- cmd.setRoleCommand(Command.START.toString());
- cmd.setServiceName(clusterName);
- cmd.setComponentName(roleName);
- cmd.setRole(roleName);
- Map hostLevelParams = new TreeMap<>();
- hostLevelParams.put(CONTAINER_ID, containerId);
- cmd.setHostLevelParams(hostLevelParams);
-
- Map roleParams = new TreeMap<>();
- cmd.setRoleParams(roleParams);
- cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart));
- startCommand = new ComponentCommand();
- startCommand.setName("START");
- startCommand.setType("docker");
- startCommand.setExec("exec");
- cmd.setCommandParams(setCommandParameters(startCommand, timeout, true));
-
- Map> configurations = buildCommandConfigurations(
- appConf, getAmState().getInternalsSnapshot(), containerId, roleName,
- roleGroup);
- Map> componentConfigurations = buildComponentConfigurations(appConf);
- cmd.setComponentConfigurations(componentConfigurations);
-
- Map dockerConfig = new HashMap();
- if (isYarnDockerContainer(roleGroup)) {
- dockerConfig.put(
- "docker.startCommand",
- getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
- "start_command"));
- cmd.setYarnDockerMode(true);
- } else {
- dockerConfig.put(
- "docker.command_path",
- getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
- "commandPath"));
-
- dockerConfig.put("docker.image_name",
- getConfigFromMetaInfo(roleGroup, "image"));
- // options should always have -d
- String options = getConfigFromMetaInfoWithAppConfigOverriding(
- roleGroup, "options");
- if(options != null && !options.isEmpty()){
- options = options + " -d";
- } else {
- options = "-d";
- }
- dockerConfig.put("docker.options", options);
- // options should always have -d
- dockerConfig.put(
- "docker.containerPort",
- getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
- "containerPort"));
- dockerConfig
- .put(
- "docker.hostPort",
- getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
- "hostPort"));
-
- dockerConfig.put(
- "docker.mounting_directory",
- getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
- "containerMount"));
- dockerConfig
- .put(
- "docker.host_mounting_directory",
- getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
- "hostMount"));
-
- dockerConfig.put("docker.additional_param",
- getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, "additionalParam"));
-
- dockerConfig.put("docker.input_file.mount_path", getConfigFromMetaInfo(
- roleGroup, "containerPath"));
- }
-
- String lifetime = getConfigFromMetaInfoWithAppConfigOverriding(
- roleGroup, "lifetime");
- dockerConfig.put("docker.lifetime", lifetime);
- configurations.put("docker", dockerConfig);
- String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(
- roleGroup, "statusCommand");
- if (statusCommand == null) {
- if(isYarnDockerContainer(roleGroup)){
- //should complain the required field is null
- } else {
- statusCommand = "docker top "
- + containerId + " | grep \"\"";
- }
- }
- dockerConfig.put("docker.status_command",statusCommand);
-
- cmd.setConfigurations(configurations);
- // configurations.get("global").put("exec_cmd", startCommand.getExec());
- cmd.addContainerDetails(roleGroup, getMetaInfo(roleGroup));
-
- log.info("Docker- command: {}", cmd.toString());
-
- response.addExecutionCommand(cmd);
- }
-
- private String getConfigFromMetaInfo(String roleGroup, String configName) {
- String result = null;
-
- List containers = getApplicationComponent(
- roleGroup).getDockerContainers();// to support multi container per
- // component later
- log.debug("Docker- containers metainfo: {}", containers.toString());
- if (containers.size() > 0) {
- DockerContainer container = containers.get(0);
- switch (configName) {
- case "start_command":
- result = container.getStartCommand();
- break;
- case "image":
- result = container.getImage();
- break;
- case "network":
- if (container.getNetwork() == null || container.getNetwork().isEmpty()) {
- result = "none";
- } else {
- result = container.getNetwork();
- }
- break;
- case "statusCommand":
- result = container.getStatusCommand();
- break;
- case "commandPath":
- result = container.getCommandPath();
- break;
- case "options":
- result = container.getOptions();
- break;
- case "containerPort":
- result = container.getPorts().size() > 0 ? container.getPorts().get(0)
- .getContainerPort() : null;// to support
- // multi port
- // later
- break;
- case "hostPort":
- result = container.getPorts().size() > 0 ? container.getPorts().get(0)
- .getHostPort() : null;// to support multi
- // port later
- break;
- case "containerMount":
- result = container.getMounts().size() > 0 ? container.getMounts()
- .get(0).getContainerMount() : null;// to support
- // multi port
- // later
- break;
- case "hostMount":
- result = container.getMounts().size() > 0 ? container.getMounts()
- .get(0).getHostMount() : null;// to support multi
- // port later
- break;
- case "additionalParam":
- result = container.getAdditionalParam();// to support multi port later
- break;
- case "runPriviledgedContainer":
- if (container.getRunPrivilegedContainer() == null) {
- result = "false";
- } else {
- result = container.getRunPrivilegedContainer();
- }
- break;
- default:
- break;
- }
- }
- log.debug("Docker- component: {} configName: {} value: {}", roleGroup, configName, result);
- return result;
- }
-
- @VisibleForTesting
- protected void addGetConfigCommand(String roleName, String roleGroup,
- String containerId, HeartBeatResponse response) throws SliderException {
- assert getAmState().isApplicationLive();
-
- StatusCommand cmd = new StatusCommand();
- String clusterName = getClusterName();
-
- cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
- cmd.setComponentName(roleName);
- cmd.setServiceName(clusterName);
- cmd.setClusterName(clusterName);
- cmd.setRoleCommand(StatusCommand.GET_CONFIG_COMMAND);
- Map hostLevelParams = new TreeMap();
- hostLevelParams.put(CONTAINER_ID, containerId);
- cmd.setHostLevelParams(hostLevelParams);
-
- hostLevelParams.put(CONTAINER_ID, containerId);
-
- response.addStatusCommand(cmd);
- }
-
- @VisibleForTesting
- protected void addStartCommand(String roleName, String roleGroup, String containerId,
- HeartBeatResponse response,
- String scriptPath, ComponentCommand startCommand,
- ComponentCommand stopCommand,
- long timeout, boolean isMarkedAutoRestart)
- throws
- SliderException {
- assert getAmState().isApplicationLive();
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
-
- ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND);
- prepareExecutionCommand(cmd);
- String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
- String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME);
- cmd.setHostname(hostName);
- cmd.setClusterName(clusterName);
- cmd.setRoleCommand(Command.START.toString());
- cmd.setServiceName(clusterName);
- cmd.setComponentName(roleName);
- cmd.setRole(roleName);
- Map hostLevelParams = new TreeMap<>();
- hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
- hostLevelParams.put(CONTAINER_ID, containerId);
- cmd.setHostLevelParams(hostLevelParams);
-
- Map roleParams = new TreeMap<>();
- cmd.setRoleParams(roleParams);
- cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart));
-
- Map> configurations =
- buildCommandConfigurations(appConf, internalsConf, containerId,
- roleName, roleGroup);
- cmd.setConfigurations(configurations);
- Map> componentConfigurations = buildComponentConfigurations(appConf);
- cmd.setComponentConfigurations(componentConfigurations);
-
- if (SliderUtils.isSet(scriptPath)) {
- cmd.setCommandParams(commandParametersSet(scriptPath, timeout, true));
- } else {
- if (startCommand == null) {
- throw new SliderException("Expected START command not found for component " + roleName);
- }
- cmd.setCommandParams(commandParametersSet(startCommand, timeout, true));
- configurations.get("global").put("exec_cmd", startCommand.getExec());
- }
-
- response.addExecutionCommand(cmd);
-
- log.debug("command looks like: {}", cmd);
- // With start command, the corresponding command for graceful stop needs to
- // be sent. This will be used when a particular container is lost as per RM,
- // but then the agent is still running and heart-beating to the Slider AM.
- ExecutionCommand cmdStop = new ExecutionCommand(
- AgentCommandType.EXECUTION_COMMAND);
- cmdStop.setTaskId(taskId.get());
- cmdStop.setCommandId(cmdStop.getTaskId() + "-1");
- cmdStop.setHostname(hostName);
- cmdStop.setClusterName(clusterName);
- cmdStop.setRoleCommand(Command.STOP.toString());
- cmdStop.setServiceName(clusterName);
- cmdStop.setComponentName(roleName);
- cmdStop.setRole(roleName);
- Map hostLevelParamsStop = new TreeMap();
- hostLevelParamsStop.put(JAVA_HOME, appConf.getGlobalOptions()
- .getOption(JAVA_HOME, ""));
- hostLevelParamsStop.put(CONTAINER_ID, containerId);
- cmdStop.setHostLevelParams(hostLevelParamsStop);
-
- Map roleParamsStop = new TreeMap();
- cmdStop.setRoleParams(roleParamsStop);
- cmdStop.getRoleParams().put("auto_restart",
- Boolean.toString(isMarkedAutoRestart));
-
- if (SliderUtils.isSet(scriptPath)) {
- cmdStop.setCommandParams(commandParametersSet(scriptPath, timeout, true));
- } else {
- if (stopCommand == null) {
- stopCommand = ComponentCommand.getDefaultComponentCommand("STOP");
- }
- cmd.setCommandParams(commandParametersSet(stopCommand, timeout, true));
- configurations.get("global").put("exec_cmd", startCommand.getExec());
- }
-
-
- Map> configurationsStop = buildCommandConfigurations(
- appConf, internalsConf, containerId, roleName, roleGroup);
- cmdStop.setConfigurations(configurationsStop);
- response.addExecutionCommand(cmdStop);
- }
-
- @VisibleForTesting
- protected void addUpgradeCommand(String roleName, String roleGroup, String containerId,
- HeartBeatResponse response, String scriptPath, long timeout)
- throws SliderException {
- assert getAmState().isApplicationLive();
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
-
- ExecutionCommand cmd = new ExecutionCommand(
- AgentCommandType.EXECUTION_COMMAND);
- prepareExecutionCommand(cmd);
- String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
- String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME);
- cmd.setHostname(hostName);
- cmd.setClusterName(clusterName);
- cmd.setRoleCommand(Command.UPGRADE.toString());
- cmd.setServiceName(clusterName);
- cmd.setComponentName(roleName);
- cmd.setRole(roleName);
- Map hostLevelParams = new TreeMap();
- hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions()
- .getMandatoryOption(JAVA_HOME));
- hostLevelParams.put(CONTAINER_ID, containerId);
- cmd.setHostLevelParams(hostLevelParams);
- cmd.setCommandParams(commandParametersSet(scriptPath, timeout, true));
-
- Map> configurations = buildCommandConfigurations(
- appConf, internalsConf, containerId, roleName, roleGroup);
- cmd.setConfigurations(configurations);
- response.addExecutionCommand(cmd);
- }
-
- protected void addStopCommand(String roleName, String roleGroup, String containerId,
- HeartBeatResponse response, String scriptPath, long timeout,
- boolean isInUpgradeMode) throws SliderException {
- assert getAmState().isApplicationLive();
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
-
- ExecutionCommand cmdStop = new ExecutionCommand(
- AgentCommandType.EXECUTION_COMMAND);
- cmdStop.setTaskId(taskId.get());
- cmdStop.setCommandId(cmdStop.getTaskId() + "-1");
- String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
- String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME);
- cmdStop.setHostname(hostName);
- cmdStop.setClusterName(clusterName);
- // Upgrade stop is differentiated by passing a transformed role command -
- // UPGRADE_STOP
- cmdStop.setRoleCommand(Command.transform(Command.STOP, isInUpgradeMode));
- cmdStop.setServiceName(clusterName);
- cmdStop.setComponentName(roleName);
- cmdStop.setRole(roleName);
- Map hostLevelParamsStop = new TreeMap();
- hostLevelParamsStop.put(JAVA_HOME, appConf.getGlobalOptions()
- .getMandatoryOption(JAVA_HOME));
- hostLevelParamsStop.put(CONTAINER_ID, containerId);
- cmdStop.setHostLevelParams(hostLevelParamsStop);
- cmdStop.setCommandParams(commandParametersSet(scriptPath, timeout, true));
-
- Map> configurationsStop = buildCommandConfigurations(
- appConf, internalsConf, containerId, roleName, roleGroup);
- cmdStop.setConfigurations(configurationsStop);
- response.addExecutionCommand(cmdStop);
- }
-
- protected static String getJDKDir() {
- File javaHome = new File(System.getProperty("java.home")).getParentFile();
- File jdkDirectory = null;
- if (javaHome.getName().contains("jdk")) {
- jdkDirectory = javaHome;
- }
- if (jdkDirectory != null) {
- return jdkDirectory.getAbsolutePath();
- } else {
- return "";
- }
- }
-
- protected Map getAllocatedPorts() {
- return getAllocatedPorts(SHARED_PORT_TAG);
- }
-
- protected Map> getComponentInstanceData() {
- return this.componentInstanceData;
- }
-
- protected Map getAllocatedPorts(String containerId) {
- if (!this.allocatedPorts.containsKey(containerId)) {
- synchronized (this.allocatedPorts) {
- if (!this.allocatedPorts.containsKey(containerId)) {
- this.allocatedPorts.put(containerId,
- new ConcurrentHashMap());
- }
- }
- }
- return this.allocatedPorts.get(containerId);
- }
-
- private Map> buildCommandConfigurations(
- ConfTreeOperations appConf, ConfTreeOperations internalsConf,
- String containerId, String roleName, String roleGroup)
- throws SliderException {
-
- Map> configurations = new TreeMap<>();
- Map tokens = providerUtils.getStandardTokenMap(appConf,
- internalsConf, roleName, roleGroup, getClusterName());
- tokens.put("${CONTAINER_ID}", containerId);
-
- Set configs = new HashSet();
- configs.addAll(getApplicationConfigurationTypes(roleGroup));
- configs.addAll(getSystemConfigurationsRequested(appConf));
-
- for (String configType : configs) {
- addNamedConfiguration(configType, appConf.getGlobalOptions().options,
- configurations, tokens, containerId, roleName,
- roleGroup);
- if (appConf.getComponent(roleGroup) != null) {
- addNamedConfiguration(configType, appConf.getComponent(roleGroup).options,
- configurations, tokens, containerId, roleName, roleGroup);
- }
- }
-
- //do a final replacement of re-used configs
- dereferenceAllConfigs(configurations);
-
- return configurations;
- }
-
- @VisibleForTesting
- protected void dereferenceAllConfigs(Map> configurations) {
- providerUtils.dereferenceAllConfigs(configurations);
- }
-
- @VisibleForTesting
- protected Set getSystemConfigurationsRequested(ConfTreeOperations appConf) {
- return providerUtils.getSystemConfigurationsRequested(appConf);
- }
-
- @VisibleForTesting
- protected List getApplicationConfigurationTypes(String roleGroup) {
- List configList = new ArrayList();
- configList.add(GLOBAL_CONFIG_TAG);
-
- List configFiles = getMetaInfo(roleGroup).getApplication().getConfigFiles();
- for (ConfigFile configFile : configFiles) {
- log.info("Expecting config type {}.", configFile.getDictionaryName());
- configList.add(configFile.getDictionaryName());
- }
- for (Component component : getMetaInfo(roleGroup).getApplication().getComponents()) {
- if (!component.getName().equals(roleGroup)) {
- continue;
- }
- if (component.getDockerContainers() == null) {
- continue;
- }
- for (DockerContainer container : component.getDockerContainers()) {
- if (container.getConfigFiles() == null) {
- continue;
- }
- for (ConfigFile configFile : container.getConfigFiles()) {
- log.info("Expecting config type {}.", configFile.getDictionaryName());
- configList.add(configFile.getDictionaryName());
- }
- }
- }
-
- // remove duplicates. mostly worried about 'global' being listed
- return new ArrayList(new HashSet(configList));
- }
-
- private void addNamedConfiguration(String configName, Map sourceConfig,
- Map> configurations,
- Map tokens, String containerId,
- String roleName, String roleGroup) {
- Map config = new HashMap();
- if (configName.equals(GLOBAL_CONFIG_TAG)) {
- addDefaultGlobalConfig(config, containerId, roleName);
- }
- // add role hosts to tokens
- addRoleRelatedTokens(tokens);
- providerUtils.propagateSiteOptions(sourceConfig, config, configName, tokens);
-
- //apply any port updates
- if (!this.getAllocatedPorts().isEmpty()) {
- for (String key : config.keySet()) {
- String value = config.get(key);
- String lookupKey = configName + "." + key;
- if (!value.contains(PER_CONTAINER_TAG)) {
- // If the config property is shared then pass on the already allocated value
- // from any container
- if (this.getAllocatedPorts().containsKey(lookupKey)) {
- config.put(key, getAllocatedPorts().get(lookupKey));
- }
- } else {
- if (this.getAllocatedPorts(containerId).containsKey(lookupKey)) {
- config.put(key, getAllocatedPorts(containerId).get(lookupKey));
- }
- }
- }
- }
-
- //apply defaults only if the key is not present and value is not empty
- if (getDefaultConfigs(roleGroup).containsKey(configName)) {
- log.info("Adding default configs for type {}.", configName);
- for (PropertyInfo defaultConfigProp : getDefaultConfigs(roleGroup).get(configName).getPropertyInfos()) {
- if (!config.containsKey(defaultConfigProp.getName())) {
- if (!defaultConfigProp.getName().isEmpty() &&
- defaultConfigProp.getValue() != null &&
- !defaultConfigProp.getValue().isEmpty()) {
- config.put(defaultConfigProp.getName(), defaultConfigProp.getValue());
- }
- }
- }
- }
-
- configurations.put(configName, config);
- }
-
- @VisibleForTesting
- protected void addRoleRelatedTokens(Map tokens) {
- providerUtils.addRoleRelatedTokens(tokens, getAmState());
- }
-
- private void addDefaultGlobalConfig(Map config, String containerId, String roleName) {
- config.put("app_log_dir", "${AGENT_LOG_ROOT}");
- config.put("app_pid_dir", "${AGENT_WORK_ROOT}/app/run");
- config.put("app_install_dir", "${AGENT_WORK_ROOT}/app/install");
- config.put("app_conf_dir", "${AGENT_WORK_ROOT}/" + APP_CONF_DIR);
- config.put("app_input_conf_dir", "${AGENT_WORK_ROOT}/" + PROPAGATED_CONF_DIR_NAME);
- config.put("app_container_id", containerId);
- config.put("app_container_tag", tags.getTag(roleName, containerId));
-
- // add optional parameters only if they are not already provided
- if (!config.containsKey("pid_file")) {
- config.put("pid_file", "${AGENT_WORK_ROOT}/app/run/component.pid");
- }
- if (!config.containsKey("app_root")) {
- config.put("app_root", "${AGENT_WORK_ROOT}/app/install");
- }
- }
-
- private void buildRoleHostDetails(Map details) {
- for (Map.Entry> entry :
- getRoleClusterNodeMapping().entrySet()) {
- details.put(entry.getKey() + " Host(s)/Container(s)",
- new MonitorDetail(providerUtils.getHostsList(
- entry.getValue().values(), false).toString(), false));
- }
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
deleted file mode 100644
index 281895a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.apache.slider.providers.ProviderRole;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class AgentRoles {
-
- /**
- * List of roles Agent provider does not have any roles by default. All roles are read from the application
- * specification.
- */
- protected static final List ROLES =
- new ArrayList();
-
- public static List getRoles() {
- return ROLES;
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
deleted file mode 100644
index 23e05a3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.providers.agent.application.metadata.AbstractMetainfoParser;
-import org.apache.slider.providers.agent.application.metadata.AddonPackageMetainfoParser;
-import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
-import org.apache.slider.providers.agent.application.metadata.DefaultConfigParser;
-import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-
-import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
-
-/**
- *
- */
-public class AgentUtils {
- private static final Logger log = LoggerFactory.getLogger(AgentUtils.class);
-
- public static Metainfo getApplicationMetainfoFromSummaryFile(
- SliderFileSystem fileSystem, String metainfoPath, boolean metainfoForAddon) {
- FileSystem fs = fileSystem.getFileSystem();
- Path appPathXML = new Path(metainfoPath + ".metainfo.xml");
- Path appPathJson = new Path(metainfoPath + ".metainfo.json");
- Path appPathUsed = null;
- try {
- FSDataInputStream appStream = null;
- if (fs.exists(appPathXML)) {
- appPathUsed = appPathXML;
- appStream = fs.open(appPathXML);
- return parseMetainfo(appStream, metainfoForAddon, "xml");
- } else if (fs.exists(appPathJson)) {
- appPathUsed = appPathJson;
- appStream = fs.open(appPathJson);
- return parseMetainfo(appStream, metainfoForAddon, "json");
- }
- } catch (IOException e) {
- log.info("Failed to get metainfo from summary file {} - {}", appPathUsed,
- e.getMessage());
- log.debug("Failed to get metainfo", e);
- }
- return null;
- }
-
- public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
- String metainfoPath, boolean metainfoForAddon) throws IOException,
- BadConfigException {
- log.info("Reading metainfo at {}", metainfoPath);
- Metainfo metainfo = getApplicationMetainfoFromSummaryFile(fileSystem,
- metainfoPath, metainfoForAddon);
- if (metainfo != null) {
- log.info("Got metainfo from summary file");
- return metainfo;
- }
-
- FileSystem fs = fileSystem.getFileSystem();
- Path appPath = new Path(metainfoPath);
-
- InputStream metainfoJsonStream = SliderUtils.getApplicationResourceInputStream(
- fs, appPath, "metainfo.json");
- if (metainfoJsonStream == null) {
- InputStream metainfoXMLStream = SliderUtils.getApplicationResourceInputStream(
- fs, appPath, "metainfo.xml");
- if (metainfoXMLStream != null) {
- metainfo = parseMetainfo(metainfoXMLStream, metainfoForAddon, "xml");
- }
- } else {
- metainfo = parseMetainfo(metainfoJsonStream, metainfoForAddon, "json");
- }
-
- if (metainfo == null) {
- log.error("metainfo is unavailable at {}.", metainfoPath);
- throw new FileNotFoundException("metainfo.xml/json is required in app package. " +
- appPath);
- }
- return metainfo;
- }
-
- private static Metainfo parseMetainfo(InputStream stream,
- boolean metainfoForAddon, String type) throws IOException {
- AbstractMetainfoParser metainfoParser = null;
- if (metainfoForAddon) {
- metainfoParser = new AddonPackageMetainfoParser();
- } else {
- metainfoParser = new MetainfoParser();
- }
- if (type.equals("xml")) {
- return metainfoParser.fromXmlStream(stream);
- } else if (type.equals("json")) {
- return metainfoParser.fromJsonStream(stream);
- }
- return null;
- }
-
- static DefaultConfig getDefaultConfig(SliderFileSystem fileSystem,
- String appDef, String configFileName)
- throws IOException {
- // this is the path inside the zip file
- String fileToRead = "configuration/" + configFileName;
- log.info("Reading default config file {} at {}", fileToRead, appDef);
- InputStream configStream = SliderUtils.getApplicationResourceInputStream(
- fileSystem.getFileSystem(), new Path(appDef), fileToRead);
- if (configStream == null) {
- log.error("{} is unavailable at {}.", fileToRead, appDef);
- throw new IOException("Expected config file " + fileToRead + " is not available.");
- }
-
- return new DefaultConfigParser().parse(configStream);
- }
-
- static String getMetainfoComponentName(String roleGroup,
- ConfTreeOperations appConf) throws BadConfigException {
- String prefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, null);
- if (prefix == null) {
- return roleGroup;
- }
- if (!roleGroup.startsWith(prefix)) {
- throw new BadConfigException("Component " + roleGroup + " doesn't start" +
- " with prefix " + prefix);
- }
- return roleGroup.substring(prefix.length());
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/Command.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/Command.java
deleted file mode 100644
index 647cb86..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/Command.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-/** The states a component instance can be. */
-public enum Command {
- NOP, // do nothing
- INSTALL, // Install the component
- INSTALL_ADDON, // Install add on packages if any
- START, // Start the component
- STOP, // Stop the component
- UPGRADE, // The component will undergo upgrade
- TERMINATE; // Send terminate signal to agent
-
- public static Command getCommand(String commandVal) {
- if (commandVal.equals(Command.START.toString())) {
- return Command.START;
- }
- if (commandVal.equals(Command.INSTALL.toString())) {
- return Command.INSTALL;
- }
- if (commandVal.equals(Command.STOP.toString())) {
- return Command.STOP;
- }
- if (commandVal.equals(Command.UPGRADE.toString())) {
- return Command.UPGRADE;
- }
- if (commandVal.equals(Command.TERMINATE.toString())) {
- return Command.TERMINATE;
- }
-
- return Command.NOP;
- }
-
- public static String transform(Command command, boolean isUpgrade) {
- switch (command) {
- case STOP:
- return isUpgrade ? "UPGRADE_STOP" : command.name();
- default:
- return command.name();
- }
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java
deleted file mode 100644
index 35d9116..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-/** Command results. */
-public enum CommandResult {
- IN_PROGRESS, // Command is in progress
- COMPLETED, // Command has successfully completed
- FAILED; // Command has failed
-
- public static CommandResult getCommandResult(String commandResVal) {
- if (commandResVal.equals(CommandResult.COMPLETED.toString())) {
- return CommandResult.COMPLETED;
- }
- if (commandResVal.equals(CommandResult.FAILED.toString())) {
- return CommandResult.FAILED;
- }
- if (commandResVal.equals(CommandResult.IN_PROGRESS.toString())) {
- return CommandResult.IN_PROGRESS;
- }
-
- throw new IllegalArgumentException("Unrecognized value " + commandResVal);
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
deleted file mode 100644
index 4abac7a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.providers.agent.application.metadata.CommandOrder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
-
-/**
- * Stores the command dependency order for all components in a service.
- * SUPERVISOR-START NIMBUS-STARTED Means, SUPERVISOR START
- * requires NIMBUS to be STARTED
- */
-public class ComponentCommandOrder {
- public static final Logger log =
- LoggerFactory.getLogger(ComponentCommandOrder.class);
- private static char SPLIT_CHAR = '-';
- Map>> dependencies =
- new HashMap>>();
- Map> prefixRoleMap = new HashMap<>();
- Map rolePrefixMap = new HashMap<>();
-
- public ComponentCommandOrder() {}
-
- public ComponentCommandOrder(List commandOrders,
- ConfTreeOperations resources) {
- mergeCommandOrders(commandOrders, resources);
- }
-
- void mergeCommandOrders(List commandOrders,
- ConfTreeOperations resources) {
- for (String component : resources.getComponentNames()) {
- String prefix = SliderUtils.trimPrefix(
- resources.getComponentOpt(component, ROLE_PREFIX, null));
- if (prefix != null) {
- rolePrefixMap.put(component, prefix);
- if (!prefixRoleMap.containsKey(prefix)) {
- prefixRoleMap.put(prefix, new HashSet());
- }
- prefixRoleMap.get(prefix).add(component);
- }
- }
- if (commandOrders != null && commandOrders.size() > 0) {
- for (CommandOrder commandOrder : commandOrders) {
- ComponentCommand componentCmd = getComponentCommand(
- commandOrder.getCommand(), resources);
- String requires = commandOrder.getRequires();
- List requiredStates = parseRequiredStates(requires,
- resources);
- if (requiredStates.size() > 0) {
- Map> compDep = dependencies.get(componentCmd.command);
- if (compDep == null) {
- compDep = new HashMap<>();
- dependencies.put(componentCmd.command, compDep);
- }
-
- List requirements = compDep.get(componentCmd.componentName);
- if (requirements == null) {
- requirements = new ArrayList<>();
- compDep.put(componentCmd.componentName, requirements);
- }
-
- requirements.addAll(requiredStates);
- }
- }
- }
- }
-
- private List parseRequiredStates(String requires,
- ConfTreeOperations resources) {
- if (requires == null || requires.length() < 2) {
- throw new IllegalArgumentException("Input cannot be null and must contain component and state.");
- }
-
- String[] componentStates = requires.split(",");
- List retList = new ArrayList();
- for (String componentStateStr : componentStates) {
- retList.add(getComponentState(componentStateStr, resources));
- }
-
- return retList;
- }
-
- private ComponentCommand getComponentCommand(String compCmdStr,
- ConfTreeOperations resources) {
- if (compCmdStr == null || compCmdStr.trim().length() < 2) {
- throw new IllegalArgumentException("Input cannot be null and must contain component and command.");
- }
-
- compCmdStr = compCmdStr.trim();
- int splitIndex = compCmdStr.lastIndexOf(SPLIT_CHAR);
- if (splitIndex == -1 || splitIndex == 0 || splitIndex == compCmdStr.length() - 1) {
- throw new IllegalArgumentException("Input does not appear to be well-formed.");
- }
- String compStr = compCmdStr.substring(0, splitIndex);
- String cmdStr = compCmdStr.substring(splitIndex + 1);
-
- if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) {
- throw new IllegalArgumentException("Component " + compStr + " specified" +
- " in command order does not exist");
- }
-
- Command cmd = Command.valueOf(cmdStr);
-
- if (cmd != Command.START) {
- throw new IllegalArgumentException("Dependency order can only be specified for START.");
- }
- return new ComponentCommand(compStr, cmd);
- }
-
- private ComponentState getComponentState(String compStStr,
- ConfTreeOperations resources) {
- if (compStStr == null || compStStr.trim().length() < 2) {
- throw new IllegalArgumentException("Input cannot be null.");
- }
-
- compStStr = compStStr.trim();
- int splitIndex = compStStr.lastIndexOf(SPLIT_CHAR);
- if (splitIndex == -1 || splitIndex == 0 || splitIndex == compStStr.length() - 1) {
- throw new IllegalArgumentException("Input does not appear to be well-formed.");
- }
- String compStr = compStStr.substring(0, splitIndex);
- String stateStr = compStStr.substring(splitIndex + 1);
-
- if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) {
- throw new IllegalArgumentException("Component " + compStr + " specified" +
- " in command order does not exist");
- }
-
- State state = State.valueOf(stateStr);
- if (state != State.STARTED && state != State.INSTALLED) {
- throw new IllegalArgumentException("Dependency order can only be specified against STARTED/INSTALLED.");
- }
- return new ComponentState(compStr, state);
- }
-
- // dependency is still on component level, but not package level
- // so use component name to check dependency, not component-package
- public boolean canExecute(String component, Command command, Collection currentStates) {
- if (!dependencies.containsKey(command)) {
- return true;
- }
- List required = new ArrayList<>();
- if (dependencies.get(command).containsKey(component)) {
- required.addAll(dependencies.get(command).get(component));
- }
- String prefix = rolePrefixMap.get(component);
- if (prefix != null && dependencies.get(command).containsKey(prefix)) {
- required.addAll(dependencies.get(command).get(prefix));
- }
-
- for (ComponentState stateToMatch : required) {
- for (ComponentInstanceState currState : currentStates) {
- log.debug("Checking schedule {} {} against dependency {} is {}",
- component, command, currState.getComponentName(), currState.getState());
- if (currState.getComponentName().equals(stateToMatch.componentName) ||
- (prefixRoleMap.containsKey(stateToMatch.componentName) &&
- prefixRoleMap.get(stateToMatch.componentName).contains(currState.getComponentName()))) {
- if (currState.getState() != stateToMatch.state) {
- if (stateToMatch.state == State.STARTED) {
- log.info("Cannot schedule {} {} as dependency {} is {}",
- component, command, currState.getComponentName(), currState.getState());
- return false;
- } else {
- //state is INSTALLED
- if (currState.getState() != State.STARTING && currState.getState() != State.STARTED) {
- log.info("Cannot schedule {} {} as dependency {} is {}",
- component, command, currState.getComponentName(), currState.getState());
- return false;
- }
- }
- }
- }
- }
- }
- return true;
- }
-
- static class ComponentState {
- public String componentName;
- public State state;
-
- public ComponentState(String componentName, State state) {
- this.componentName = componentName;
- this.state = state;
- }
- }
-
- static class ComponentCommand {
- public String componentName;
- public Command command;
-
- public ComponentCommand(String componentName, Command command) {
- this.componentName = componentName;
- this.command = command;
- }
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
deleted file mode 100644
index 6ee0ebb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
+++ /dev/null
@@ -1,340 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import java.util.Map;
-import java.util.TreeMap;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.providers.agent.application.metadata.Component;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/** This class implements a simple state machine for component instances. */
-public class ComponentInstanceState {
- public static final Logger log =
- LoggerFactory.getLogger(ComponentInstanceState.class);
- private static int MAX_FAILURE_TOLERATED = 3;
- private static String INVALID_TRANSITION_ERROR =
- "Result %s for command %s is not expected for component %s in state %s.";
-
- private final String componentName;
- private final ContainerId containerId;
- private final String containerIdAsString;
- private final String applicationId;
- private State state = State.INIT;
- private State targetState = State.STARTED;
- private int failuresSeen = 0;
- private Boolean configReported = false;
- private long lastHeartbeat = 0;
- private String ip;
- private String hostname;
- private ContainerState containerState;
-
- private Map pkgStatuses;
- private String nextPkgToInstall;
-
- private boolean stopInitiated;
-
- public ComponentInstanceState(String componentName,
- ContainerId containerId,
- String applicationId) {
- this(componentName, containerId, applicationId,
- new TreeMap());
- }
-
- public ComponentInstanceState(String componentName,
- ContainerId containerId,
- String applicationId, Map pkgStatuses) {
- this.componentName = componentName;
- this.containerId = containerId;
- this.containerIdAsString = containerId.toString();
- this.applicationId = applicationId;
- this.containerState = ContainerState.INIT;
- this.lastHeartbeat = System.currentTimeMillis();
- this.pkgStatuses = pkgStatuses;
- }
-
- public String getComponentName() {
- return componentName;
- }
-
- public Boolean getConfigReported() {
- return configReported;
- }
-
- public void setConfigReported(Boolean configReported) {
- this.configReported = configReported;
- }
-
- public ContainerState getContainerState() {
- return containerState;
- }
-
- public void setContainerState(ContainerState containerState) {
- this.containerState = containerState;
- }
-
- public long getLastHeartbeat() {
- return lastHeartbeat;
- }
-
- /**
- * Update the heartbeat, and change container state
- * to mark as healthy if appropriate
- * @param heartbeatTime last time the heartbeat was seen
- * @return the current container state
- */
- public ContainerState heartbeat(long heartbeatTime) {
- this.lastHeartbeat = heartbeatTime;
- if(containerState == ContainerState.UNHEALTHY ||
- containerState == ContainerState.INIT) {
- containerState = ContainerState.HEALTHY;
- }
- return containerState;
- }
-
-
- public ContainerId getContainerId() {
- return containerId;
- }
-
- public void commandIssued(Command command) {
- commandIssued(command, false);
- }
-
- public void commandIssued(Command command, boolean isInUpgradeMode) {
- Command expected = getNextCommand(isInUpgradeMode);
- if (expected != command) {
- throw new IllegalArgumentException("Command " + command + " is not allowed in state " + state);
- }
- if (expected == Command.INSTALL_ADDON) {
- // for add on packages, the pkg must be nextPkgToInstall
- State currentState = pkgStatuses.get(nextPkgToInstall);
- log.debug("Command issued: component: {} is in {}", componentName,
- currentState);
- State nextState = currentState.getNextState(command);
- pkgStatuses.put(nextPkgToInstall, nextState);
- log.debug("Command issued: component: {} is now in {}", componentName,
- nextState);
- } else {
- // for master package
- state = state.getNextState(command);
- }
- }
-
- public void applyCommandResult(CommandResult result, Command command,
- String pkg) {
- // if the heartbeat is for a package
- // update that package's state in the component status
- // and don't bother with the master pkg
- if (StringUtils.isNotEmpty(pkg)
- && !Component.MASTER_PACKAGE_NAME.equals(pkg)) {
- log.debug("This result is for component: {} pkg: {}", componentName, pkg);
- State previousPkgState = pkgStatuses.get(pkg);
- log.debug("Currently component: {} pkg: {} is in state: {}",
- componentName, pkg, previousPkgState.toString());
- State nextPkgState = previousPkgState.getNextState(result);
- pkgStatuses.put(pkg, nextPkgState);
- log.debug("Component: {} pkg: {} next state: {}", componentName, pkg,
- nextPkgState);
- } else {
- log.debug("This result is for component: {} master package",
- componentName);
- applyCommandResult(result, command);
- }
- }
-
- public void applyCommandResult(CommandResult result, Command command) {
- if (!this.state.couldHaveIssued(command)) {
- throw new IllegalStateException("Invalid command " + command + " for state " + this.state);
- }
-
- try {
- if (result == CommandResult.FAILED) {
- failuresSeen++;
- } else if (result == CommandResult.COMPLETED) {
- failuresSeen = 0;
- }
- state = state.getNextState(result);
- } catch (IllegalArgumentException e) {
- String message = String.format(INVALID_TRANSITION_ERROR,
- result.toString(),
- command.toString(),
- componentName,
- state.toString());
- log.warn(message);
- throw new IllegalStateException(message);
- }
- }
-
- public boolean hasPendingCommand() {
- if (state.canIssueCommands() &&
- state != targetState &&
- failuresSeen < MAX_FAILURE_TOLERATED) {
- return true;
- }
-
- return false;
- }
-
- public Command getNextCommand() {
- return getNextCommand(false);
- }
-
- public Command getNextCommand(boolean isInUpgradeMode) {
- if (!hasPendingCommand()) {
- nextPkgToInstall = null;
- return Command.NOP;
- }
-
- log.debug("In getNextCommand, checking for component: {} ", componentName);
- // if the master pkg is just installed, check if any add on pkg need to be
- // installed
- nextPkgToInstall = null;
- if (state == State.INSTALLED) {
- for (Map.Entry pkgStatus : pkgStatuses.entrySet()) {
- String pkg = pkgStatus.getKey();
- State pkgState = pkgStatus.getValue();
- log.debug("In getNextCommand, pkg: {} is in {}", pkg, pkgState);
- if (pkgState == State.INSTALLING) {
- // first check if any pkg is install in progress, if so, wait
- // so we don't need to do anything, just return NOP
- log.debug("In getNextCommand, pkg: {} we are issuing NOP", pkg);
- nextPkgToInstall = pkg;
- return Command.NOP;
- } else if (pkgState == State.INIT) {
- // temporarily storing pkg here
- // in case no pkg in 'installing' state
- // will return the package to install
- nextPkgToInstall = pkg;
- }
- }
- // when we reach here, no pkg is in 'installing' state
- if (nextPkgToInstall != null) {
- // nextPkgToInstall != null means some pkg is in INIT state
- // issue 'install' to the pkg we have stored in nextPkgToInstall
- log.debug("In getNextCommand, pkg: {} we are issuing install addon",
- nextPkgToInstall);
- return Command.INSTALL_ADDON;
- }
- }
- return this.state.getSupportedCommand(isInUpgradeMode, stopInitiated);
- }
-
- public State getState() {
- return state;
- }
-
- @VisibleForTesting
- protected void setState(State state) {
- this.state = state;
- }
-
- public State getTargetState() {
- return targetState;
- }
-
- public void setTargetState(State targetState) {
- this.targetState = targetState;
- }
-
- public String getNextPkgToInstall() {
- return nextPkgToInstall;
- }
-
- public boolean isStopInitiated() {
- return stopInitiated;
- }
-
- public void setStopInitiated(boolean stopInitiated) {
- this.stopInitiated = stopInitiated;
- }
-
- @Override
- public int hashCode() {
- int hashCode = 1;
-
- hashCode = hashCode ^ (componentName != null ? componentName.hashCode() : 0);
- hashCode = hashCode ^ (containerIdAsString != null ? containerIdAsString.hashCode() : 0);
- hashCode = hashCode ^ (applicationId != null ? applicationId.hashCode() : 0);
- return hashCode;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
-
- if (o == null || getClass() != o.getClass()) return false;
-
- ComponentInstanceState that = (ComponentInstanceState) o;
-
- if (this.componentName != null ?
- !this.componentName.equals(that.componentName) : this.componentName != null) {
- return false;
- }
-
- if (this.containerIdAsString != null ?
- !this.containerIdAsString.equals(that.containerIdAsString) : this.containerIdAsString != null) {
- return false;
- }
-
- if (this.applicationId != null ?
- !this.applicationId.equals(that.applicationId) : this.applicationId != null) {
- return false;
- }
-
- return true;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("ComponentInstanceState{");
- sb.append("containerIdAsString='").append(containerIdAsString).append('\'');
- sb.append(", state=").append(state);
- sb.append(", failuresSeen=").append(failuresSeen);
- sb.append(", lastHeartbeat=").append(lastHeartbeat);
- sb.append(", containerState=").append(containerState);
- sb.append(", componentName='").append(componentName).append('\'');
- sb.append(", ip=").append(ip);
- sb.append(", hostname='").append(hostname).append('\'');
- sb.append('}');
- return sb.toString();
- }
-
- public String getIp() {
- return ip;
- }
-
- public void setIp(String ip) {
- this.ip = ip;
- }
-
- public String getHostname() {
- return hostname;
- }
-
- public void setHostname(String hostname) {
- this.hostname = hostname;
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java
deleted file mode 100644
index 68f63fa..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.ConcurrentHashMap;
-
-/** A simple tag provider that attempts to associate tags from 1-N to all container of a component */
-public class ComponentTagProvider {
- private static final Logger log = LoggerFactory.getLogger(ComponentTagProvider.class);
- private static String FREE = "free";
- private final ConcurrentHashMap> allTags;
-
- public ComponentTagProvider() {
- allTags = new ConcurrentHashMap>();
- }
-
- /**
- * Record an assigned tag to a container
- *
- * @param component
- * @param containerId
- * @param tag
- */
- public void recordAssignedTag(String component, String containerId, String tag) {
- if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
- Integer key = null;
- try {
- key = Integer.valueOf(tag);
- } catch (NumberFormatException nfe) {
- //ignore
- }
- if (key != null && key > 0) {
- ConcurrentHashMap compTags = getComponentSpecificTags(component);
- synchronized (compTags) {
- for (int index = 1; index <= key.intValue(); index++) {
- String tempKey = new Integer(index).toString();
- if (!compTags.containsKey(tempKey)) {
- compTags.put(tempKey, FREE);
- }
- }
- compTags.put(key.toString(), containerId);
- }
- }
- }
- }
-
- /**
- * Get a tag for container
- *
- * @param component
- * @param containerId
- *
- * @return
- */
- public String getTag(String component, String containerId) {
- if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
- ConcurrentHashMap compTags = getComponentSpecificTags(component);
- synchronized (compTags) {
- for (String key : compTags.keySet()) {
- if (compTags.get(key).equals(containerId)) {
- return key;
- }
- }
- for (String key : compTags.keySet()) {
- if (compTags.get(key).equals(FREE)) {
- compTags.put(key, containerId);
- return key;
- }
- }
- String newKey = new Integer(compTags.size() + 1).toString();
- compTags.put(newKey, containerId);
- return newKey;
- }
- }
- return "";
- }
-
- /**
- * Release a tag associated with a container
- *
- * @param component
- * @param containerId
- */
- public void releaseTag(String component, String containerId) {
- if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
- ConcurrentHashMap compTags = allTags.get(component);
- if (compTags != null) {
- synchronized (compTags) {
- for (String key : compTags.keySet()) {
- if (compTags.get(key).equals(containerId)) {
- compTags.put(key, FREE);
- }
- }
- }
- }
- }
- }
-
- private ConcurrentHashMap getComponentSpecificTags(String component) {
- if (!allTags.containsKey(component)) {
- synchronized (allTags) {
- if (!allTags.containsKey(component)) {
- allTags.put(component, new ConcurrentHashMap());
- }
- }
- }
- return allTags.get(component);
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java
deleted file mode 100644
index 0394ba2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-/** The states a component instance can be. */
-public enum ContainerState {
- INIT, // Container is not net activated
- HEALTHY, // Agent is heartbeating
- UNHEALTHY, // Container is unhealthy - no heartbeat for some interval
- HEARTBEAT_LOST; // Container is lost - request a new instance
-
- /**
- * Indicates whether or not it is a valid state to produce a command.
- *
- * @return true if command can be issued for this state.
- */
- public boolean canIssueCommands() {
- switch (this) {
- case HEALTHY:
- return true;
- default:
- return false;
- }
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
deleted file mode 100644
index 4293916..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-
-/** Monitors the container state and heartbeats. */
-public class HeartbeatMonitor implements Runnable {
- protected static final Logger log =
- LoggerFactory.getLogger(HeartbeatMonitor.class);
- private final int threadWakeupInterval; //1 minute
- private final AgentProviderService provider;
- private volatile boolean shouldRun = true;
- private Thread monitorThread = null;
-
- public HeartbeatMonitor(AgentProviderService provider, int threadWakeupInterval) {
- this.provider = provider;
- this.threadWakeupInterval = threadWakeupInterval;
- }
-
- public void shutdown() {
- shouldRun = false;
- }
-
- public void start() {
- log.info("Starting heartbeat monitor with interval {}", threadWakeupInterval);
- monitorThread = new Thread(this);
- monitorThread.start();
- }
-
- void join(long millis) throws InterruptedException {
- if (isAlive()) {
- monitorThread.join(millis);
- }
- }
-
- public boolean isAlive() {
- return monitorThread != null && monitorThread.isAlive();
- }
-
- @Override
- public void run() {
- while (shouldRun) {
- try {
- log.debug("Putting monitor to sleep for " + threadWakeupInterval + " " +
- "milliseconds");
- Thread.sleep(threadWakeupInterval);
- doWork(System.currentTimeMillis());
- } catch (InterruptedException ex) {
- log.warn("Scheduler thread is interrupted going to stop", ex);
- shouldRun = false;
- } catch (Exception ex) {
- log.warn("Exception received", ex);
- } catch (Throwable t) {
- log.warn("ERROR", t);
- }
- }
- }
-
- /**
- * Every interval the current state of the container are checked. If the state is INIT or HEALTHY and no HB are
- * received in last check interval they are marked as UNHEALTHY. INIT is when the agent is started but it did not
- * communicate at all. HEALTHY being the AM has received heartbeats. After an interval as UNHEALTHY the container is
- * declared unavailable
- * @param now current time in milliseconds ... tests can set this explicitly
- */
- @VisibleForTesting
- public void doWork(long now) {
- Map componentStatuses = provider.getComponentStatuses();
- if (componentStatuses != null) {
- for (String containerLabel : componentStatuses.keySet()) {
- ComponentInstanceState componentInstanceState = componentStatuses.get(containerLabel);
- long timeSinceLastHeartbeat = now - componentInstanceState.getLastHeartbeat();
-
- if (timeSinceLastHeartbeat > threadWakeupInterval) {
- switch (componentInstanceState.getContainerState()) {
- case INIT:
- case HEALTHY:
- componentInstanceState.setContainerState(ContainerState.UNHEALTHY);
- log.warn(
- "Component {} marked UNHEALTHY. Last heartbeat received at {} approx. {} ms. back.",
- componentInstanceState,
- componentInstanceState.getLastHeartbeat(),
- timeSinceLastHeartbeat);
- break;
- case UNHEALTHY:
- if (timeSinceLastHeartbeat > threadWakeupInterval * 2) {
- componentInstanceState.setContainerState(
- ContainerState.HEARTBEAT_LOST);
- log.warn(
- "Component {} marked HEARTBEAT_LOST. Last heartbeat received at {} approx. {} ms. back.",
- componentInstanceState, componentInstanceState.getLastHeartbeat(),
- timeSinceLastHeartbeat);
- ContainerId containerId =
- componentInstanceState.getContainerId();
- provider.lostContainer(containerLabel, containerId);
- }
- break;
- case HEARTBEAT_LOST:
- // unexpected case
- log.warn("Heartbeat from lost component: {}", componentInstanceState);
- break;
- }
-
- }
- }
- }
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/State.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/State.java
deleted file mode 100644
index 5603f8d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/State.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-/** The states a component instance can be. */
-public enum State {
- INIT, // Not installed
- INSTALLING, // Being installed
- INSTALLED, // Installed (or stopped)
- STARTING, // Starting
- STARTED, // Started
- INSTALL_FAILED, // Install failed, start failure in INSTALLED
- UPGRADING, // Undergoing upgrade, perform necessary pre-upgrade steps
- UPGRADED, // Pre-upgrade steps completed
- STOPPING, // Stop has been issued
- STOPPED, // Agent has stopped
- TERMINATING; // Terminate signal to ask the agent to kill itself
- // No need for state TERMINATED (as the agent is dead by then)
-
- /**
- * Indicates whether or not it is a valid state to produce a command.
- *
- * @return true if command can be issued for this state.
- */
- public boolean canIssueCommands() {
- switch (this) {
- case INSTALLING:
- case STARTING:
- case UPGRADING:
- case STOPPING:
- case TERMINATING:
- return false;
- default:
- return true;
- }
- }
-
- /**
- * Returns valid command in this state.
- *
- * @return command allowed in this state.
- */
- public Command getSupportedCommand() {
- return getSupportedCommand(false);
- }
-
- public Command getSupportedCommand(boolean isInUpgradeMode) {
- return getSupportedCommand(isInUpgradeMode, false);
- }
-
- public Command getSupportedCommand(boolean isInUpgradeMode,
- boolean stopInitiated) {
- switch (this) {
- case INIT:
- case INSTALL_FAILED:
- return Command.INSTALL;
- case INSTALLED:
- return Command.START;
- case STARTED:
- return isInUpgradeMode ? Command.UPGRADE : (stopInitiated) ? Command.STOP
- : Command.NOP;
- case UPGRADED:
- return Command.STOP;
- case STOPPED:
- return Command.TERMINATE;
- default:
- return Command.NOP;
- }
- }
-
- /**
- * Returns next state based on the command result.
- *
- * @return next state.
- */
- public State getNextState(CommandResult result) throws IllegalArgumentException {
- switch (result) {
- case IN_PROGRESS:
- if (this == State.INSTALLING || this == State.STARTING
- || this == State.UPGRADING || this == State.STOPPING
- || this == State.TERMINATING) {
- return this;
- } else {
- throw new IllegalArgumentException(result + " is not valid for " + this);
- }
- case COMPLETED:
- if (this == State.INSTALLING) {
- return State.INSTALLED;
- } else if (this == State.STARTING) {
- return State.STARTED;
- } else if (this == State.UPGRADING) {
- return State.UPGRADED;
- } else if (this == State.STOPPING) {
- return State.STOPPED;
- } else {
- throw new IllegalArgumentException(result + " is not valid for " + this);
- }
- case FAILED:
- if (this == State.INSTALLING) {
- return State.INSTALL_FAILED;
- } else if (this == State.STARTING) {
- return State.INSTALLED;
- } else if (this == State.UPGRADING) {
- // if pre-upgrade failed, force stop now, so mark it upgraded
- // what other options can be exposed to app owner?
- return State.UPGRADED;
- } else if (this == State.STOPPING) {
- // if stop fails, force mark it stopped (and let container terminate)
- return State.STOPPED;
- } else if (this == State.STOPPED) {
- // if in stopped state, force mark it as terminating
- return State.TERMINATING;
- } else {
- throw new IllegalArgumentException(result + " is not valid for " + this);
- }
- default:
- throw new IllegalArgumentException("Bad command result " + result);
- }
- }
-
- /**
- * Returns next state based on the command.
- *
- * @return next state.
- */
- public State getNextState(Command command) throws IllegalArgumentException {
- switch (command) {
- case INSTALL:
- if (this == State.INIT || this == State.INSTALL_FAILED) {
- return State.INSTALLING;
- } else {
- throw new IllegalArgumentException(command + " is not valid for " + this);
- }
- case INSTALL_ADDON:
- if (this == State.INIT || this == State.INSTALL_FAILED) {
- return State.INSTALLING;
- } else {
- throw new IllegalArgumentException(command + " is not valid for " + this);
- }
- case START:
- if (this == State.INSTALLED) {
- return State.STARTING;
- } else {
- throw new IllegalArgumentException(command + " is not valid for " + this);
- }
- case UPGRADE:
- if (this == State.STARTED) {
- return State.UPGRADING;
- } else {
- throw new IllegalArgumentException(command + " is not valid for " + this);
- }
- case STOP:
- if (this == State.STARTED || this == State.UPGRADED) {
- return State.STOPPING;
- } else {
- throw new IllegalArgumentException(command + " is not valid for " + this);
- }
- case TERMINATE:
- if (this == State.STOPPED) {
- return State.TERMINATING;
- } else {
- throw new IllegalArgumentException(command + " is not valid for " + this);
- }
- case NOP:
- return this;
- default:
- throw new IllegalArgumentException("Bad command " + command);
- }
- }
-
- public boolean couldHaveIssued(Command command) {
- if ((this == State.INSTALLING && command == Command.INSTALL)
- || (this == State.STARTING && command == Command.START)
- || (this == State.UPGRADING && command == Command.UPGRADE)
- || (this == State.STOPPING
- && (command == Command.STOP || command == Command.NOP))
- || (this == State.TERMINATING && command == Command.TERMINATE)
- ) {
- return true;
- }
- return false;
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java
deleted file mode 100644
index b6ae4de..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.codehaus.jackson.annotate.JsonProperty;
-
-/**
- * Component defined in master package metainfo.json
- */
-public abstract class AbstractComponent implements Validate {
- public static final String TYPE_STANDARD = "STANDARD";
- public static final String TYPE_DOCKER = "DOCKER";
- public static final String TYPE_PYTHON = "PYTHON";
- public static final String CATEGORY_MASTER = "MASTER";
- public static final String CATEGORY_SLAVE = "SLAVE";
- public static final String CATEGORY_CLIENT = "CLIENT";
- public static final String MASTER_PACKAGE_NAME = "MASTER";
-
- protected String name;
- protected CommandScript commandScript;
- protected List commands = new ArrayList<>();
-
- public AbstractComponent() {
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public CommandScript getCommandScript() {
- return commandScript;
- }
-
- public void addCommandScript(CommandScript commandScript) {
- this.commandScript = commandScript;
- }
-
- @JsonProperty("commands")
- public List getCommands() {
- return commands;
- }
-
- public void setCommands(List commands) {
- this.commands = commands;
- }
-
- public void addCommand(ComponentCommand command) {
- commands.add(command);
- }
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder("{");
- sb.append("\n\"name\": ").append(name);
- sb.append(",\n\"commandScript\" :").append(commandScript);
- sb.append('}');
- return sb.toString();
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java
deleted file mode 100644
index 67d1f15..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-
-import org.apache.commons.digester.Digester;
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.xml.sax.SAXException;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-
-/**
- * This abstract class provide common functionality to parse metainfo.json for
- * either master package or add on packages.
- */
-public abstract class AbstractMetainfoParser {
- protected final GsonBuilder gsonBuilder = new GsonBuilder();
- protected final Gson gson;
- private static final Logger log = LoggerFactory
- .getLogger(AbstractMetainfoParser.class);
-
- public AbstractMetainfoParser() {
- gson = gsonBuilder.create();
- }
-
- /**
- * Convert to a JSON string
- *
- * @return a JSON string description
- *
- * @throws IOException Problems mapping/writing the object
- */
- public String toJsonString(Metainfo metaInfo) throws IOException {
- return gson.toJson(metaInfo);
- }
-
- /**
- * Convert from JSON
- *
- * @param json input
- *
- * @return the parsed JSON
- *
- * @throws IOException IO
- */
- public Metainfo fromJsonString(String json)
- throws IOException {
- return gson.fromJson(json, Metainfo.class);
- }
-
- /**
- * Parse metainfo from an IOStream
- *
- * @param is
- *
- * @return
- *
- * @throws IOException
- */
- public Metainfo fromJsonStream(InputStream is) throws IOException {
- log.debug("loading from xml stream");
- StringWriter writer = new StringWriter();
- IOUtils.copy(is, writer);
- return fromJsonString(writer.toString());
- }
-
- /**
- * Parse metainfo from an XML formatted IOStream
- *
- * @param metainfoStream
- *
- * @return
- *
- * @throws IOException
- */
- public Metainfo fromXmlStream(InputStream metainfoStream) throws IOException {
- log.debug("loading from xml stream");
- Digester digester = new Digester();
- digester.setValidating(false);
-
- composeSchema(digester);
-
- try {
- return (Metainfo) digester.parse(metainfoStream);
- } catch (IOException e) {
- log.debug("IOException in metainfoparser during fromXmlStream: "
- + e.getMessage());
- } catch (SAXException e) {
- log.debug("SAXException in metainfoparser during fromXmlStream: "
- + e.getMessage());
- } finally {
- if (metainfoStream != null) {
- metainfoStream.close();
- }
- }
-
- return null;
- }
-
- /**
- * Compose the schema for the metainfo
- *
- * @param Digester - The Digester object we passed in to compose the schema
- *
- * @return
- *
- * @throws IOException
- */
- abstract protected void composeSchema(Digester digester);
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java
deleted file mode 100644
index cfa2895..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.codehaus.jackson.annotate.JsonProperty;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Application type defined in the metainfo
- */
-public abstract class AbstractMetainfoSchema implements Validate {
- protected String name;
- protected String comment;
- protected String version;
- protected List configFiles = new ArrayList<>();
-
- public AbstractMetainfoSchema() {
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getComment() {
- return comment;
- }
-
- public void setComment(String comment) {
- this.comment = comment;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
- public void addConfigFile(ConfigFile configFile) {
- this.configFiles.add(configFile);
- }
-
- @JsonProperty("configFiles")
- public List getConfigFiles() {
- return configFiles;
- }
-
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java
deleted file mode 100644
index c75837f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.commons.digester.Digester;
-
-/**
- *
- */
-public class AddonPackageMetainfoParser extends AbstractMetainfoParser {
-
- protected void composeSchema(Digester digester) {
- digester.addObjectCreate("metainfo", Metainfo.class);
- digester.addBeanPropertySetter("metainfo/schemaVersion");
-
- digester.addObjectCreate("*/applicationPackage", ApplicationPackage.class);
- digester.addBeanPropertySetter("*/applicationPackage/name");
- digester.addBeanPropertySetter("*/applicationPackage/comment");
- digester.addBeanPropertySetter("*/applicationPackage/version");
-
- digester.addObjectCreate("*/component", ComponentsInAddonPackage.class);
- digester.addBeanPropertySetter("*/component/name");
- digester.addSetNext("*/component", "addComponent");
-
- digester.addObjectCreate("*/commandScript", CommandScript.class);
- digester.addBeanPropertySetter("*/commandScript/script");
- digester.addBeanPropertySetter("*/commandScript/scriptType");
- digester.addBeanPropertySetter("*/commandScript/timeout");
- digester.addSetNext("*/commandScript", "addCommandScript");
-
- digester.addObjectCreate("*/configFile", ConfigFile.class);
- digester.addBeanPropertySetter("*/configFile/type");
- digester.addBeanPropertySetter("*/configFile/fileName");
- digester.addBeanPropertySetter("*/configFile/dictionaryName");
- digester.addSetNext("*/configFile", "addConfigFile");
-
- digester.addSetRoot("*/applicationPackage", "setApplicationPackage");
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
deleted file mode 100644
index 5556c7f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.codehaus.jackson.annotate.JsonIgnore;
-import org.codehaus.jackson.annotate.JsonProperty;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Application type defined in the metainfo
- */
-public class Application extends AbstractMetainfoSchema {
- String exportedConfigs;
- List exportGroups = new ArrayList<>();
- List osSpecifics = new ArrayList<>();
- List commandOrders = new ArrayList<>();
- List packages = new ArrayList<>();
- private List components = new ArrayList<>();
-
- public Application() {
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getComment() {
- return comment;
- }
-
- public void setComment(String comment) {
- this.comment = comment;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
- public String getExportedConfigs() {
- return exportedConfigs;
- }
-
- public void setExportedConfigs(String exportedConfigs) {
- this.exportedConfigs = exportedConfigs;
- }
-
- public void addConfigFile(ConfigFile configFile) {
- this.configFiles.add(configFile);
- }
-
- @JsonProperty("configFiles")
- public List getConfigFiles() {
- return configFiles;
- }
-
- public void addComponent(Component component) {
- components.add(component);
- }
-
- @JsonProperty("components")
- public List getComponents() {
- return components;
- }
-
- public void addExportGroup(ExportGroup exportGroup) {
- exportGroups.add(exportGroup);
- }
-
- @JsonProperty("exportGroups")
- public List getExportGroups() {
- return exportGroups;
- }
-
- public void addOSSpecific(OSSpecific osSpecific) {
- osSpecifics.add(osSpecific);
- }
-
- @JsonIgnore
- public List getOSSpecifics() {
- return osSpecifics;
- }
-
- public void addCommandOrder(CommandOrder commandOrder) {
- commandOrders.add(commandOrder);
- }
-
- @JsonProperty("commandOrders")
- public List