diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 968a90b..ebac8da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -311,5 +311,12 @@ String APP_RESOURCES = "application.resources"; String APP_RESOURCES_DIR = "app/resources"; - String APP_PACKAGES_DIR = "app/packages"; + String APP_INSTALL_DIR = "app/install"; + + String OUT_FILE = "stdout.txt"; + String ERR_FILE = "stderr.txt"; + + String QUICK_LINKS = "quicklinks"; + + String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java index df174f5..4a11067 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java @@ -41,10 +41,6 @@ public AbstractClientProvider(Configuration conf) { super(conf); } - public abstract String getName(); - - public abstract List getRoles(); - /** * Generates a fixed format of application tags given one or more of * application name, version and description. This allows subsequent query for diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java new file mode 100644 index 0000000..a331c33 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers; + +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.ContainerState; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.exceptions.SliderException; +import org.apache.slider.core.launch.CommandLineBuilder; +import org.apache.slider.core.launch.ContainerLauncher; +import org.apache.slider.core.registry.docstore.PublishedConfiguration; +import org.apache.slider.server.appmaster.state.RoleInstance; +import org.apache.slider.server.appmaster.state.StateAccessForProviders; +import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher; +import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; +import java.util.Map.Entry; + +import static org.apache.slider.util.ServiceApiUtil.$; + +public abstract class AbstractProviderService extends AbstractService + implements ProviderService, SliderKeys { + + protected static final Logger log = + LoggerFactory.getLogger(AbstractProviderService.class); + private static final ProviderUtils providerUtils = new ProviderUtils(log); + protected StateAccessForProviders amState; + protected YarnRegistryViewForProviders yarnRegistry; + private ServiceTimelinePublisher serviceTimelinePublisher; + + protected AbstractProviderService(String name) { + super(name); + } + + public abstract void processArtifact(ContainerLauncher launcher, Component + component, SliderFileSystem fileSystem) throws IOException; + + @Override + public void setAMState(StateAccessForProviders stateAccessor) { + this.amState = stateAccessor; + } + + @Override + public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) { + this.yarnRegistry = yarnRegistry; + } + + public void buildContainerLaunchContext(ContainerLauncher launcher, + Application application, Container container, ProviderRole providerRole, + SliderFileSystem fileSystem, RoleInstance roleInstance) + throws IOException, SliderException { + Component component = providerRole.component; + processArtifact(launcher, component, fileSystem); + + // Generate tokens (key-value pair) for config substitution. + // Get pre-defined tokens + Map globalTokens = amState.getGlobalSubstitutionTokens(); + Map tokensForSubstitution = providerUtils + .initCompTokensForSubstitute(roleInstance); + tokensForSubstitution.putAll(globalTokens); + // Set the environment variables in launcher + launcher.putEnv(SliderUtils + .buildEnvMap(component.getConfiguration(), tokensForSubstitution)); + launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$()); + launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR); + if (System.getenv(HADOOP_USER_NAME) != null) { + launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME)); + } + launcher.setEnv("LANG", "en_US.UTF-8"); + launcher.setEnv("LC_ALL", "en_US.UTF-8"); + launcher.setEnv("LANGUAGE", "en_US.UTF-8"); + + for (Entry entry : launcher.getEnv().entrySet()) { + tokensForSubstitution.put($(entry.getKey()), entry.getValue()); + } + providerUtils.addComponentHostTokens(tokensForSubstitution, amState); + + // create config file on hdfs and add local resource + providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem, + component, tokensForSubstitution, roleInstance, amState); + + // substitute launch command + String launchCommand = ProviderUtils + .substituteStrWithTokens(component.getLaunchCommand(), + tokensForSubstitution); + CommandLineBuilder operation = new CommandLineBuilder(); + operation.add(launchCommand); + operation.addOutAndErrFiles(OUT_FILE, ERR_FILE); + launcher.addCommand(operation.build()); + + // publish exports + providerUtils + .substituteMapWithTokens(application.getQuicklinks(), tokensForSubstitution); + PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS, + application.getQuicklinks().entrySet()); + amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf); + if (serviceTimelinePublisher != null) { + serviceTimelinePublisher.serviceAttemptUpdated(application); + } + } + + public boolean processContainerStatus(ContainerId containerId, + ContainerStatus status) { + log.debug("Handling container status: {}", status); + if (SliderUtils.isEmpty(status.getIPs()) || + SliderUtils.isUnset(status.getHost())) { + return true; + } + RoleInstance instance = amState.getOwnedContainer(containerId); + if (instance == null) { + // container is completed? + return false; + } + + providerUtils.updateServiceRecord(amState, yarnRegistry, + containerId.toString(), instance.role, status.getIPs(), status.getHost()); + // TODO publish ip and host + org.apache.slider.api.resource.Container container = + instance.providerRole.component.getContainer(containerId.toString()); + if (container != null) { + container.setIp(StringUtils.join(",", status.getIPs())); + container.setHostname(status.getHost()); + container.setState(ContainerState.READY); + } else { + log.warn(containerId + " not found in Application!"); + } + return false; + } + + @Override + public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) { + this.serviceTimelinePublisher = publisher; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java index 5dd4a32..c0001ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java @@ -24,7 +24,7 @@ import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.core.exceptions.BadClusterStateException; import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.providers.agent.AgentKeys; +import org.apache.slider.providers.docker.DockerKeys; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,7 +33,7 @@ */ public abstract class SliderProviderFactory extends Configured { - public static final String DEFAULT_CLUSTER_TYPE = AgentKeys.PROVIDER_AGENT; + public static final String DEFAULT_CLUSTER_TYPE = DockerKeys.PROVIDER_DOCKER; protected static final Logger log = LoggerFactory.getLogger(SliderProviderFactory.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java index 1d5d8a0..419ff05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java @@ -20,34 +20,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.slider.common.SliderKeys; import org.apache.slider.providers.AbstractClientProvider; -import org.apache.slider.providers.ProviderRole; -import org.apache.slider.providers.ProviderUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.List; public class DockerClientProvider extends AbstractClientProvider implements SliderKeys { - protected static final Logger log = - LoggerFactory.getLogger(DockerClientProvider.class); - private static final ProviderUtils providerUtils = new ProviderUtils(log); - protected static final String NAME = "docker"; - public DockerClientProvider(Configuration conf) { super(conf); } - @Override - public String getName() { - return NAME; - } - - @Override - public List getRoles() { - return Collections.emptyList(); - } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java index 0e1d288..7413ebb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java @@ -27,7 +27,4 @@ String DEFAULT_DOCKER_NETWORK = "bridge"; Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false; - - String OUT_FILE = "stdout.txt"; - String ERR_FILE = "stderr.txt"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java index 12c2b04..f35d4d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java @@ -17,152 +17,31 @@ */ package org.apache.slider.providers.docker; -import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.Component; -import org.apache.slider.api.resource.ContainerState; -import org.apache.slider.common.SliderKeys; import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.launch.CommandLineBuilder; import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.slider.core.registry.docstore.PublishedConfiguration; -import org.apache.slider.providers.ProviderRole; -import org.apache.slider.providers.ProviderService; -import org.apache.slider.providers.ProviderUtils; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher; -import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; +import org.apache.slider.providers.AbstractProviderService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Map; -import java.util.Map.Entry; -import static org.apache.slider.util.ServiceApiUtil.$; - -public class DockerProviderService extends AbstractService - implements ProviderService, DockerKeys, SliderKeys { +public class DockerProviderService extends AbstractProviderService + implements DockerKeys { protected static final Logger log = LoggerFactory.getLogger(DockerProviderService.class); - private static final ProviderUtils providerUtils = new ProviderUtils(log); - private static final String QUICK_LINKS = "quicklinks"; - protected StateAccessForProviders amState; - protected YarnRegistryViewForProviders yarnRegistry; - private ServiceTimelinePublisher serviceTimelinePublisher; protected DockerProviderService() { - super("DockerProviderService"); - } - - @Override - public void setAMState(StateAccessForProviders stateAccessor) { - this.amState = stateAccessor; - } - - @Override - public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) { - this.yarnRegistry = yarnRegistry; + super(DockerProviderService.class.getSimpleName()); } - - public void buildContainerLaunchContext(ContainerLauncher launcher, - Application application, Container container, ProviderRole providerRole, - SliderFileSystem fileSystem, RoleInstance roleInstance) - throws IOException, SliderException { - Component component = providerRole.component; + public void processArtifact(ContainerLauncher launcher, Component + component, SliderFileSystem fileSystem) throws IOException { launcher.setYarnDockerMode(true); launcher.setDockerImage(component.getArtifact().getId()); launcher.setDockerNetwork(component.getConfiguration() .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK)); launcher.setRunPrivilegedContainer(component.getRunPrivilegedContainer()); - - // Generate tokens (key-value pair) for config substitution. - // Get pre-defined tokens - Map globalTokens = amState.getGlobalSubstitutionTokens(); - Map tokensForSubstitution = providerUtils - .initCompTokensForSubstitute(roleInstance); - tokensForSubstitution.putAll(globalTokens); - // Set the environment variables in launcher - launcher.putEnv(SliderUtils - .buildEnvMap(component.getConfiguration(), tokensForSubstitution)); - launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$()); - launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR); - if (System.getenv(HADOOP_USER_NAME) != null) { - launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME)); - } - launcher.setEnv("LANG", "en_US.UTF-8"); - launcher.setEnv("LC_ALL", "en_US.UTF-8"); - launcher.setEnv("LANGUAGE", "en_US.UTF-8"); - - for (Entry entry : launcher.getEnv().entrySet()) { - tokensForSubstitution.put($(entry.getKey()), entry.getValue()); - } - providerUtils.addComponentHostTokens(tokensForSubstitution, amState); - - // create config file on hdfs and add local resource - providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem, - component, tokensForSubstitution, roleInstance, amState); - - // substitute launch command - String launchCommand = ProviderUtils - .substituteStrWithTokens(component.getLaunchCommand(), - tokensForSubstitution); - CommandLineBuilder operation = new CommandLineBuilder(); - operation.add(launchCommand); - operation.addOutAndErrFiles(OUT_FILE, ERR_FILE); - launcher.addCommand(operation.build()); - - // publish exports - providerUtils - .substituteMapWithTokens(application.getQuicklinks(), tokensForSubstitution); - PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS, - application.getQuicklinks().entrySet()); - amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf); - if (serviceTimelinePublisher != null) { - serviceTimelinePublisher.serviceAttemptUpdated(application); - } - } - - public boolean processContainerStatus(ContainerId containerId, - ContainerStatus status) { - log.debug("Handling container status: {}", status); - if (SliderUtils.isEmpty(status.getIPs()) || - SliderUtils.isUnset(status.getHost())) { - return true; - } - RoleInstance instance = amState.getOwnedContainer(containerId); - if (instance == null) { - // container is completed? - return false; - } - - providerUtils.updateServiceRecord(amState, yarnRegistry, - containerId.toString(), instance.role, status.getIPs(), status.getHost()); - // TODO publish ip and host - org.apache.slider.api.resource.Container container = - instance.providerRole.component.getContainer(containerId.toString()); - if (container != null) { - container.setIp(StringUtils.join(",", status.getIPs())); - container.setHostname(status.getHost()); - container.setState(ContainerState.READY); - } else { - log.warn(containerId + " not found in Application!"); - } - return false; - } - - @Override - public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) { - this.serviceTimelinePublisher = publisher; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java new file mode 100644 index 0000000..9383752 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.tarball; + +import org.apache.hadoop.conf.Configuration; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.providers.AbstractClientProvider; + +public class TarballClientProvider extends AbstractClientProvider + implements SliderKeys { + + public TarballClientProvider(Configuration conf) { + super(conf); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java new file mode 100644 index 0000000..0f1ac5a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.tarball; + +import org.apache.hadoop.conf.Configuration; +import org.apache.slider.providers.AbstractClientProvider; +import org.apache.slider.providers.ProviderService; +import org.apache.slider.providers.SliderProviderFactory; + +public class TarballProviderFactory extends SliderProviderFactory { + + public TarballProviderFactory() { + } + + public TarballProviderFactory(Configuration conf) { + super(conf); + } + + @Override + public AbstractClientProvider createClientProvider() { + return new TarballClientProvider(getConf()); + } + + @Override + public ProviderService createServerProvider() { + return new TarballProviderService(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java new file mode 100644 index 0000000..65a55f0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.tarball; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.slider.api.resource.Component; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.core.launch.ContainerLauncher; +import org.apache.slider.providers.AbstractProviderService; + +import java.io.IOException; + +public class TarballProviderService extends AbstractProviderService { + + protected TarballProviderService() { + super(TarballProviderService.class.getSimpleName()); + } + + @Override + public void processArtifact(ContainerLauncher launcher, Component + component, SliderFileSystem fileSystem) throws IOException { + Path artifact = new Path(component.getArtifact().getId()); + if (!fileSystem.isFile(artifact)) { + throw new IOException("Package doesn't exist as a resource: " + + artifact.toString()); + } + log.info("Adding resource {}", artifact.toString()); + LocalResourceType type = LocalResourceType.ARCHIVE; + LocalResource packageResource = fileSystem.createAmResource( + artifact, type); + launcher.addLocalResource(APP_INSTALL_DIR, packageResource); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java index 7c096c7..1dd7836b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java @@ -23,11 +23,12 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Artifact; import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderService; -import org.apache.slider.providers.agent.AgentKeys; import org.apache.slider.server.appmaster.actions.ActionStartContainer; import org.apache.slider.server.appmaster.actions.QueueAccess; import org.apache.slider.server.appmaster.state.ContainerAssignment; @@ -43,6 +44,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import static org.apache.slider.common.SliderKeys.KEY_CONTAINER_LAUNCH_DELAY; + /** * A service for launching containers */ @@ -62,8 +65,8 @@ /** * Provider building up the command */ - private final ProviderService provider; - + private final Map providers; + /** * Filesystem to use for the launch */ @@ -75,16 +78,17 @@ /** * Construct an instance of the launcher * @param queueAccess - * @param provider the provider + * @param providers the providers * @param fs filesystem * @param envVars environment variables */ - public RoleLaunchService(QueueAccess queueAccess, ProviderService provider, + public RoleLaunchService(QueueAccess queueAccess, Map providers, SliderFileSystem fs, Map envVars) { super(ROLE_LAUNCH_SERVICE); this.actionQueue = queueAccess; this.fs = fs; - this.provider = provider; + this.providers = providers; this.envVars = envVars; } @@ -167,11 +171,17 @@ public void run() { instance.roleId = role.id; instance.environment = envDescription; + ProviderService provider = providers.get(role.component.getArtifact() + .getType()); + if (provider == null) { + throw new SliderException("Provider not initialized for " + role + .component.getArtifact().getType() + "; unexpected"); + } provider.buildContainerLaunchContext(containerLauncher, application, container, role, fs, instance); long delay = role.component.getConfiguration() - .getPropertyLong(AgentKeys.KEY_CONTAINER_LAUNCH_DELAY, 0); + .getPropertyLong(KEY_CONTAINER_LAUNCH_DELAY, 0); long maxDelay = getConfig() .getLong(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index ae03b45..2bc10ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -87,6 +87,7 @@ import org.apache.slider.api.proto.Messages; import org.apache.slider.api.proto.SliderClusterAPI; import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Artifact; import org.apache.slider.api.resource.Component; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; @@ -171,6 +172,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -342,7 +344,8 @@ * ProviderService of this cluster */ @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized") - private ProviderService providerService; + private Map providers = + new EnumMap<>(Artifact.TypeEnum.class); /** * The YARN registry service @@ -603,12 +606,20 @@ private int createAndRunCluster(String appName) throws Throwable { // obtain security state // set the global security flag for the instance definition - //get our provider - SliderProviderFactory factory = - SliderProviderFactory.createSliderProviderFactory("docker"); - providerService = factory.createServerProvider(); - // init the provider BUT DO NOT START IT YET - initAndAddService(providerService); + // initialize our providers + for (Component component : application.getComponents()) { + Artifact.TypeEnum providerType = component.getArtifact().getType(); + if (providers.containsKey(providerType)) { + continue; + } + log.info("Loading service provider type {}", providerType); + SliderProviderFactory factory = SliderProviderFactory + .createSliderProviderFactory(providerType.toString()); + ProviderService providerService = factory.createServerProvider(); + // init the provider BUT DO NOT START IT YET + initAndAddService(providerService); + providers.put(providerType, providerService); + } InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf); log.info("RM is at {}", rmSchedulerAddress); @@ -667,7 +678,9 @@ private int createAndRunCluster(String appName) throws Throwable { serviceTimelinePublisher.init(getConfig()); serviceTimelinePublisher.start(); - providerService.setServiceTimelinePublisher(serviceTimelinePublisher); + for (ProviderService providerService : providers.values()) { + providerService.setServiceTimelinePublisher(serviceTimelinePublisher); + } appState.setServiceTimelinePublisher(serviceTimelinePublisher); log.info("ServiceTimelinePublisher started."); } @@ -707,7 +720,7 @@ private int createAndRunCluster(String appName) throws Throwable { WebAppApiImpl webAppApi = new WebAppApiImpl( stateForProviders, - providerService, registryOperations, + registryOperations, metricsAndMonitoring, actionQueues); initAMFilterOptions(serviceConf); @@ -843,13 +856,15 @@ private int createAndRunCluster(String appName) throws Throwable { //launcher service launchService = new RoleLaunchService(actionQueues, - providerService, + providers, fs, envVars); deployChildService(launchService); //Give the provider access to the state, and AM - providerService.setAMState(stateForProviders); + for (ProviderService providerService : providers.values()) { + providerService.setAMState(stateForProviders); + } // chaos monkey maybeStartMonkey(); @@ -1119,7 +1134,9 @@ public void registerServiceInstance(String instanceName, SliderKeys.APP_TYPE, instanceName, appAttemptID); - providerService.bindToYarnRegistry(yarnRegistryOperations); + for (ProviderService providerService : providers.values()) { + providerService.bindToYarnRegistry(yarnRegistryOperations); + } // Yarn registry ServiceRecord serviceRecord = new ServiceRecord(); @@ -1875,7 +1892,9 @@ public void updateBlacklist(List blacklistAdditions, protected synchronized void launchProviderService() throws IOException, SliderException { // didn't start, so don't register - providerService.start(); + for (ProviderService providerService : providers.values()) { + providerService.start(); + } // and send the started event ourselves eventCallbackEvent(null); } @@ -1975,19 +1994,23 @@ public void onContainerStatusReceived(ContainerId containerId, ContainerStatus containerStatus) { LOG_YARN.debug("Container Status: id={}, status={}", containerId, containerStatus); + RoleInstance cinfo = appState.getOwnedContainer(containerId); + if (cinfo == null) { + LOG_YARN.error("Owned container not found for {}", containerId); + return; + } + ProviderService providerService = providers.get(cinfo.providerRole + .component.getArtifact().getType()); if (providerService.processContainerStatus(containerId, containerStatus)) { try { Thread.sleep(1000); } catch (InterruptedException e) { } - RoleInstance cinfo = appState.getOwnedContainer(containerId); - if (cinfo != null) { - LOG_YARN.info("Re-requesting status for role {}, {}", - cinfo.role, containerId); - //trigger another async container status - nmClientAsync.getContainerStatusAsync(containerId, - cinfo.container.getNodeId()); - } + LOG_YARN.info("Re-requesting status for role {}, {}", + cinfo.role, containerId); + //trigger another async container status + nmClientAsync.getContainerStatusAsync(containerId, + cinfo.container.getNodeId()); } else if (timelineServiceEnabled) { RoleInstance instance = appState.getOwnedContainer(containerId); if (instance != null) { @@ -2013,11 +2036,6 @@ public void onStopContainerError(ContainerId containerId, Throwable t) { LOG_YARN.warn("Failed to stop Container {}", containerId); } - - public ProviderService getProviderService() { - return providerService; - } - /** * Queue an action for immediate execution in the executor thread * @param action action to execute diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java index 094726d..02f3f0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java @@ -17,7 +17,6 @@ package org.apache.slider.server.appmaster.web; import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.slider.providers.ProviderService; import org.apache.slider.server.appmaster.actions.QueueAccess; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.state.AppState; @@ -34,11 +33,6 @@ StateAccessForProviders getAppState(); /** - * The {@link ProviderService} for the current cluster - */ - ProviderService getProviderService(); - - /** * Registry operations accessor * @return registry access */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java index fd9381c..f88f501 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java @@ -17,7 +17,6 @@ package org.apache.slider.server.appmaster.web; import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.slider.providers.ProviderService; import org.apache.slider.server.appmaster.actions.QueueAccess; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.state.StateAccessForProviders; @@ -33,21 +32,18 @@ private static final Logger log = LoggerFactory.getLogger(WebAppApiImpl.class); protected final StateAccessForProviders appState; - protected final ProviderService provider; private final RegistryOperations registryOperations; private final MetricsAndMonitoring metricsAndMonitoring; private final QueueAccess queues; public WebAppApiImpl(StateAccessForProviders appState, - ProviderService provider, RegistryOperations registryOperations, + RegistryOperations registryOperations, MetricsAndMonitoring metricsAndMonitoring, QueueAccess queues) { checkNotNull(appState); - checkNotNull(provider); this.queues = queues; this.registryOperations = registryOperations; this.appState = appState; - this.provider = provider; this.metricsAndMonitoring = metricsAndMonitoring; } @@ -57,11 +53,6 @@ public StateAccessForProviders getAppState() { } @Override - public ProviderService getProviderService() { - return provider; - } - - @Override public RegistryOperations getRegistryOperations() { return registryOperations; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java index 82d7c8f..5f44bda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java @@ -19,7 +19,6 @@ package org.apache.slider.server.appmaster.web.view; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; -import org.apache.slider.providers.ProviderService; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.appmaster.web.WebAppApi; import org.apache.slider.server.appmaster.web.rest.RestPaths; @@ -33,12 +32,10 @@ public abstract class SliderHamletBlock extends HtmlBlock { protected final StateAccessForProviders appState; - protected final ProviderService providerService; protected final RestPaths restPaths = new RestPaths(); public SliderHamletBlock(WebAppApi slider) { this.appState = slider.getAppState(); - this.providerService = slider.getProviderService(); } protected String rootPath(String absolutePath) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml index 96bfe0f..2e14918 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml @@ -24,7 +24,11 @@ true - slider.provider.docker + slider.provider.DOCKER org.apache.slider.providers.docker.DockerProviderFactory + + slider.provider.TARBALL + org.apache.slider.providers.tarball.TarballProviderFactory + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.java index e4e344e..c6c32a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.java @@ -45,11 +45,9 @@ public void setup() throws Exception { ProviderAppState providerAppState = new ProviderAppState( "undefined", appState); - ProviderService providerService = new MockProviderService(); WebAppApiImpl inst = new WebAppApiImpl( providerAppState, - providerService, null, null, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.java index 92f8559..a07695f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.java @@ -65,14 +65,12 @@ @Before public void setup() throws Exception { super.setup(); - ProviderService providerService = new MockProviderService(); ProviderAppState providerAppState = new ProviderAppState( "undefined", appState); WebAppApiImpl inst = new WebAppApiImpl( providerAppState, - providerService, null, METRICS, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestIndexBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestIndexBlock.java index 96eb3d7..cf25311 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestIndexBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestIndexBlock.java @@ -58,14 +58,12 @@ public void setup() throws Exception { super.setup(); assertNotNull(appState); - ProviderService providerService = new MockProviderService(); ProviderAppState providerAppState = new ProviderAppState( "undefined", appState); WebAppApiImpl inst = new WebAppApiImpl( providerAppState, - providerService, null, METRICS, null);