From dc2df1b2e83b0076ad291cea0ccd828c5d3bf407 Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Mon, 9 Apr 2018 13:21:40 +0530 Subject: [PATCH] YARN-8126 --- .../service/client/SystemServiceManagerImpl.java | 22 ++- ...Impl.java => TestSystemServiceManagerImpl.java} | 12 +- .../resources/system-services/bad/bad.yarnfile | 16 +++ .../sync/user1/example-app1.yarnfile | 0 .../sync/user1/example-app2.yarnfile | 0 .../sync/user1/example-app3.json | 0 .../sync/user2/example-app1.yarnfile | 0 .../sync/user2/example-app2.yarnfile | 0 .../yarn/service/TestSystemServiceManager.java | 156 --------------------- .../site/markdown/yarn-service/Configurations.md | 1 + 10 files changed, 40 insertions(+), 167 deletions(-) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/{TestSystemServiceImpl.java => TestSystemServiceManagerImpl.java} (93%) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/{users => system-services}/sync/user1/example-app1.yarnfile (100%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/{users => system-services}/sync/user1/example-app2.yarnfile (100%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/{users => system-services}/sync/user1/example-app3.json (100%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/{users => system-services}/sync/user2/example-app1.yarnfile (100%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/{users => system-services}/sync/user2/example-app2.yarnfile (100%) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestSystemServiceManager.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java index 225f8bdfedb..9e45962d091 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java @@ -92,10 +92,12 @@ private Thread serviceLaucher; @VisibleForTesting - private int skipCounter; + private int badFileNameExtensionSkipCounter; @VisibleForTesting private Map ignoredUserServices = new HashMap<>(); + @VisibleForTesting + private int badDirSkipCounter; public SystemServiceManagerImpl() { super(SystemServiceManagerImpl.class.getName()); @@ -268,6 +270,7 @@ void scanForUserServices() throws IOException { } else if (launchType.getPath().getName().equals(ASYNC)) { scanForUserServiceDefinition(launchType.getPath(), asyncUserServices); } else { + badDirSkipCounter++; LOG.debug("Scanner skips for unknown dir {}.", launchType.getPath()); } } @@ -308,7 +311,7 @@ private void scanForUserServiceDefinition(Path userDirPath, if (!filename.endsWith(YARN_FILE_SUFFIX)) { LOG.info("Scanner skips for unknown file extension, filename = {}", filename); - skipCounter++; + badFileNameExtensionSkipCounter++; continue; } Service service = getServiceDefinition(serviceCache.getPath()); @@ -325,9 +328,10 @@ private void scanForUserServiceDefinition(Path userDirPath, LOG.warn( "Ignoring service {} for the user {} as it is already present," + " filename = {}", service.getName(), userName, filename); + } else { + LOG.info("Added service {} for the user {}, filename = {}", + service.getName(), userName, filename); } - LOG.info("Added service {} for the user {}, filename = {}", - service.getName(), userName, filename); } } } @@ -375,7 +379,13 @@ private Service getServiceDefinition(Path filePath) { return syncUserServices; } - @VisibleForTesting int getSkipCounter() { - return skipCounter; + @VisibleForTesting + int getBadFileNameExtensionSkipCounter() { + return badFileNameExtensionSkipCounter; + } + + @VisibleForTesting + int getbadDirSkipCounter() { + return badDirSkipCounter; } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java similarity index 93% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceImpl.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java index 27632f9956a..fea16a8a0b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java @@ -41,13 +41,13 @@ /** * Test class for system service manager. */ -public class TestSystemServiceImpl { +public class TestSystemServiceManagerImpl { private static final Logger LOG = - LoggerFactory.getLogger(TestSystemServiceImpl.class); + LoggerFactory.getLogger(TestSystemServiceManagerImpl.class); private SystemServiceManagerImpl systemService; private Configuration conf; - private String resourcePath = "users"; + private String resourcePath = "system-services"; private String[] users = new String[] {"user1", "user2"}; private static Map> loadedServices = new HashMap<>(); @@ -88,7 +88,9 @@ public void testSystemServiceSubmission() throws Exception { ignoredUserServices.containsKey(users[0])); int count = ignoredUserServices.get(users[0]); Assert.assertEquals(1, count); - Assert.assertEquals(1, systemService.getSkipCounter()); + Assert.assertEquals(1, + systemService.getBadFileNameExtensionSkipCounter()); + Assert.assertEquals(1, systemService.getbadDirSkipCounter()); Map> userServices = systemService.getSyncUserServices(); @@ -112,7 +114,7 @@ private void verifyForScannedUserServices( while (iterator.hasNext()) { Service next = iterator.next(); Assert.assertTrue( - "Service name doesn't exist in expected " + "userService " + "Service name doesn't exist in expected userService " + serviceNames, serviceNames.contains(next.getName())); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile new file mode 100644 index 00000000000..1d514d665fb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/bad/bad.yarnfile @@ -0,0 +1,16 @@ +{ + "name": "bad", + "version": "1.0.0", + "components" : + [ + { + "name": "simple", + "number_of_containers": 1, + "launch_command": "sleep 2", + "resource": { + "cpus": 1, + "memory": "128" + } + } + ] +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app1.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app1.yarnfile similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app1.yarnfile rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app1.yarnfile diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app2.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app2.yarnfile similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app2.yarnfile rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app2.yarnfile diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app3.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app3.json similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user1/example-app3.json rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user1/example-app3.json diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user2/example-app1.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app1.yarnfile similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user2/example-app1.yarnfile rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app1.yarnfile diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user2/example-app2.yarnfile b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app2.yarnfile similarity index 100% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/users/sync/user2/example-app2.yarnfile rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/resources/system-services/sync/user2/example-app2.yarnfile diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestSystemServiceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestSystemServiceManager.java deleted file mode 100644 index dbff02f5aab..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestSystemServiceManager.java +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.service.api.records.Artifact; -import org.apache.hadoop.yarn.service.api.records.ComponentState; -import org.apache.hadoop.yarn.service.api.records.Service; -import org.apache.hadoop.yarn.service.api.records.ServiceState; -import org.apache.hadoop.yarn.service.exceptions.SliderException; -import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders; -import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; - -import java.io.IOException; -import java.util.Map; - -import static org.mockito.Mockito.mock; - -/** - * Tests for {@link ServiceManager}. - */ -public class TestSystemServiceManager { - - @Rule - public ServiceTestUtils.ServiceFSWatcher rule = - new ServiceTestUtils.ServiceFSWatcher(); - - @Test - public void testUpgrade() throws IOException, SliderException { - ServiceManager serviceManager = createTestServiceManager("testUpgrade"); - upgrade(serviceManager, "v2", false); - Assert.assertEquals("service not upgraded", ServiceState.UPGRADING, - serviceManager.getServiceSpec().getState()); - } - - @Test - public void testRestartNothingToUpgrade() - throws IOException, SliderException { - ServiceManager serviceManager = createTestServiceManager("testRestart"); - upgrade(serviceManager, "v2", false); - - //make components stable - serviceManager.getServiceSpec().getComponents().forEach(comp -> { - comp.setState(ComponentState.STABLE); - }); - serviceManager.handle(new ServiceEvent(ServiceEventType.START)); - Assert.assertEquals("service not re-started", ServiceState.STABLE, - serviceManager.getServiceSpec().getState()); - } - - @Test - public void testRestartWithPendingUpgrade() - throws IOException, SliderException { - ServiceManager serviceManager = createTestServiceManager("testRestart"); - upgrade(serviceManager, "v2", true); - serviceManager.handle(new ServiceEvent(ServiceEventType.START)); - Assert.assertEquals("service should still be upgrading", - ServiceState.UPGRADING, serviceManager.getServiceSpec().getState()); - } - - - private void upgrade(ServiceManager service, String version, - boolean upgradeArtifact) - throws IOException, SliderException { - Service upgradedDef = ServiceTestUtils.createExampleApplication(); - upgradedDef.setName(service.getName()); - upgradedDef.setVersion(version); - if (upgradeArtifact) { - Artifact upgradedArtifact = createTestArtifact("2"); - upgradedDef.getComponents().forEach(component -> { - component.setArtifact(upgradedArtifact); - }); - } - writeUpgradedDef(upgradedDef); - ServiceEvent upgradeEvent = new ServiceEvent(ServiceEventType.UPGRADE); - upgradeEvent.setVersion("v2"); - service.handle(upgradeEvent); - } - - private ServiceManager createTestServiceManager(String name) - throws IOException { - ServiceContext context = new ServiceContext(); - context.service = createBaseDef(name); - context.fs = rule.getFs(); - - context.scheduler = new ServiceScheduler(context) { - @Override - protected YarnRegistryViewForProviders createYarnRegistryOperations( - ServiceContext context, RegistryOperations registryClient) { - return mock(YarnRegistryViewForProviders.class); - } - }; - - context.scheduler.init(rule.getConf()); - - Map - componentState = context.scheduler.getAllComponents(); - context.service.getComponents().forEach(component -> { - componentState.put(component.getName(), - new org.apache.hadoop.yarn.service.component.Component(component, - 1L, context)); - }); - return new ServiceManager(context); - } - - static Service createBaseDef(String name) { - ApplicationId applicationId = ApplicationId.newInstance( - System.currentTimeMillis(), 1); - Service serviceDef = ServiceTestUtils.createExampleApplication(); - serviceDef.setId(applicationId.toString()); - serviceDef.setName(name); - serviceDef.setState(ServiceState.STARTED); - Artifact artifact = createTestArtifact("1"); - - serviceDef.getComponents().forEach(component -> - component.setArtifact(artifact)); - return serviceDef; - } - - static Artifact createTestArtifact(String artifactId) { - Artifact artifact = new Artifact(); - artifact.setId(artifactId); - artifact.setType(Artifact.TypeEnum.TARBALL); - return artifact; - } - - private void writeUpgradedDef(Service upgradedDef) - throws IOException, SliderException { - Path upgradePath = rule.getFs().buildClusterUpgradeDirPath( - upgradedDef.getName(), upgradedDef.getVersion()); - ServiceApiUtil.createDirAndPersistApp(rule.getFs(), upgradePath, - upgradedDef); - } - -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md index 7ec2ecb879d..71d8d173673 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md @@ -127,6 +127,7 @@ Above config make the service AM to be retried at max 10 times. |yarn.service.rolling-log.include-pattern| The regex expression for including log files whose file name matches it when aggregating the logs while app is running. |yarn.service.rolling-log.exclude-pattern| The regex expression for excluding log files whose file name matches it when aggregating the logs while app is running. If the log file name matches both include and exclude pattern, this file will be excluded. |yarn.service.container-recovery.timeout.ms| The timeout in milliseconds after which the service AM releases all the containers of previous attempt which are not yet recovered by the RM. By default, it is set to 120000, i.e. 2 minutes. +|yarn.service.system-service.dir| FS directory path to load configured system services. There are 2 modes to start these services i.e sync or async. Sync mode services are started along with RM start. This might delay a bit RM transition to active period. Async mode is services are started in separate thread so RM transition to active is bit faster. Note: Hierarchy of system services to configure is <sys-dir-path>/sync or async/<users>/<service-files-with-dot-yarnfile-extension>. Example: <sys-dir-path>/sync/user1/hbase.yarnfile; | ## Constant variables for custom service The service framework provides some constant variables for user to configure their services. These variables are either dynamically generated by the system or are static ones such as service name defined by the user. -- 2.13.6 (Apple Git-96)