From de77bf11e431c0cd63cce99bb240ee985b2ca1b9 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Thu, 20 Feb 2020 16:15:01 +0100 Subject: [PATCH] YARN-10130. FS-CS converter: Do not allow output dir to be the same as input dir --- .../FSConfigToCSConfigArgumentHandler.java | 48 ++++++++++++++ .../FSConfigConverterTestCommons.java | 2 +- ...TestFSConfigToCSConfigArgumentHandler.java | 64 +++++++++++++++++++ 3 files changed, 113 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java index c751953f7bc..68ed1bd6468 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java @@ -25,6 +25,7 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +40,13 @@ private static final Logger LOG = LoggerFactory.getLogger(FSConfigToCSConfigArgumentHandler.class); + private static final String ALREADY_CONTAINS_EXCEPTION_MSG = + "The %s (provided with %s|%s arguments) contains " + + "the %s provided with the %s|%s options."; + private static final String ALREADY_CONTAINS_FILE_EXCEPTION_MSG = + "The %s %s (provided with %s|%s arguments) already contains a file " + + "or directory named %s which will be the output of the conversion!"; + private FSConfigToCSConfigRuleHandler ruleHandler; private FSConfigToCSConfigConverterParams converterParams; private ConversionOptions conversionOptions; @@ -213,6 +221,7 @@ private FSConfigToCSConfigConverterParams validateInputFiles( checkFile(CliOption.FAIR_SCHEDULER, fairSchedulerXmlFile); checkFile(CliOption.CONVERSION_RULES, conversionRulesFile); checkDirectory(CliOption.OUTPUT_DIR, outputDir); + checkOutputDirDoesNotContainXmls(yarnSiteXmlFile, outputDir); return FSConfigToCSConfigConverterParams.Builder.create() .withYarnSiteXmlConfig(yarnSiteXmlFile) @@ -225,6 +234,45 @@ private FSConfigToCSConfigConverterParams validateInputFiles( .build(); } + private static void checkOutputDirDoesNotContainXmls(String yarnSiteXmlFile, + String outputDir) { + if (yarnSiteXmlFile == null || outputDir == null) { + return; + } + + // check whether yarn-site.xml is not in the output folder + File xmlFile = new File(yarnSiteXmlFile); + File xmlParentFolder = xmlFile.getParentFile(); + File output = new File(outputDir); + if (output.equals(xmlParentFolder)) { + throw new IllegalArgumentException( + String.format(ALREADY_CONTAINS_EXCEPTION_MSG, + CliOption.OUTPUT_DIR.name, CliOption.OUTPUT_DIR.shortSwitch, + CliOption.OUTPUT_DIR.longSwitch, CliOption.YARN_SITE.name, + CliOption.YARN_SITE.shortSwitch, + CliOption.YARN_SITE.longSwitch)); + } + + // check whether the output folder does not contain nor yarn-site.xml + // neither capacity-scheduler.xml + checkFileNotInOutputDir(output, + YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); + checkFileNotInOutputDir(output, + YarnConfiguration.CS_CONFIGURATION_FILE); + } + + private static void checkFileNotInOutputDir(File output, String fileName) { + File xmlChild = new File(output, fileName); + if (xmlChild.exists()) { + throw new IllegalArgumentException( + String.format(ALREADY_CONTAINS_FILE_EXCEPTION_MSG, + CliOption.OUTPUT_DIR.name, output, + CliOption.OUTPUT_DIR.shortSwitch, + CliOption.OUTPUT_DIR.longSwitch, + fileName)); + } + } + private void printHelp(Options opts) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("General options are: ", opts); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java index 4f678095f08..af915d459f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java @@ -35,7 +35,7 @@ * */ public class FSConfigConverterTestCommons { - private final static String TEST_DIR = + public final static String TEST_DIR = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); public final static String FS_ALLOC_FILE = new File(TEST_DIR, "test-fair-scheduler.xml").getAbsolutePath(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java index edf4c86ceb2..277ced9ccfd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java @@ -18,12 +18,16 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.List; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -492,4 +496,64 @@ public void testEnabledTerminalRuleCheck() throws Exception { assertFalse("No terminal rule check was enabled", conversionOptions.isNoRuleTerminalCheck()); } + + @Test + public void testYarnSiteOptionInOutputFolder() throws Exception { + setupFSConfigConversionFiles(true); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = new String[] { + "-y", FSConfigConverterTestCommons.YARN_SITE_XML, + "-o", FSConfigConverterTestCommons.TEST_DIR}; + + int retVal = argumentHandler.parseAndConvert(args); + assertEquals("Return value", -1, retVal); + + assertTrue(fsTestCommons.getErrContent() + .toString().contains("contains the yarn-site.xml")); + } + + private void testFileExistsInOutputFolder(String file) throws Exception { + File testFile = new File(FSConfigConverterTestCommons.OUTPUT_DIR, file); + try { + FileUtils.touch(testFile); + + setupFSConfigConversionFiles(true); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = new String[] { + "-y", FSConfigConverterTestCommons.YARN_SITE_XML, + "-o", FSConfigConverterTestCommons.OUTPUT_DIR}; + + int retVal = argumentHandler.parseAndConvert(args); + assertEquals("Return value", -1, retVal); + + // TODO remove + assertNull(fsTestCommons.getErrContent().toString()); + + assertTrue(fsTestCommons.getErrContent() + .toString().contains("x")); + } finally { + if (testFile.exists()) { + testFile.delete(); + } + } + } + + @Test + public void testYarnSiteExistsInOutputFolder() throws Exception { + testFileExistsInOutputFolder( + YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); + } + + @Test + public void testCapacitySchedulerXmlExistsInOutputFolder() + throws Exception { + testFileExistsInOutputFolder( + YarnConfiguration.CS_CONFIGURATION_FILE); + } } \ No newline at end of file -- 2.21.0