diff --git itests/bin/generate-cli-splits.sh itests/bin/generate-cli-splits.sh new file mode 100755 index 0000000..2e5959b --- /dev/null +++ itests/bin/generate-cli-splits.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +usage() { + echo "$0 " + exit 1 +} + +[ "$1" == "" ] && usage +[ "$2" == "" ] && usage + + +inDir="$1" +outDir="$2" + +git grep SplitSupport.process | grep "$1" | cut -d ':' -f1|while read f;do + + echo "processing: $f" + n="`grep N_SPLITS "$f" |cut -d= -f2|tr -c -d '0-9'`" + echo " * nSplits: $n" + + for((i=0;i $oDir/`basename $f` + } +done diff --git itests/qtest-spark/pom.xml itests/qtest-spark/pom.xml index 60d032d..19f1f81 100644 --- itests/qtest-spark/pom.xml +++ itests/qtest-spark/pom.xml @@ -417,5 +417,53 @@ - + + + qsplits + + + + org.apache.maven.plugins + maven-antrun-plugin + + + generate-split-tests + generate-sources + + + + + + + + + + + run + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-test-source + generate-sources + + add-test-source + + + + target/generated-test-sources + + + + + + + + + diff --git itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestMiniSparkOnYarnCliDriver.java itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestMiniSparkOnYarnCliDriver.java index 889d0f2..c19d4db 100644 --- itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestMiniSparkOnYarnCliDriver.java +++ itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestMiniSparkOnYarnCliDriver.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - + package org.apache.hadoop.hive.cli; import java.io.File; @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hadoop.hive.cli.control.SplitSupport; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -34,11 +35,13 @@ @RunWith(Parameterized.class) public class TestMiniSparkOnYarnCliDriver { + private static final int N_SPLITS = 5; + static CliAdapter adapter = new CliConfigs.SparkOnYarnCliConfig().getCliAdapter(); @Parameters(name = "{0}") public static List getParameters() throws Exception { - return adapter.getParameters(); + return SplitSupport.process(adapter.getParameters(), TestMiniSparkOnYarnCliDriver.class, N_SPLITS); } @ClassRule diff --git itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkCliDriver.java itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkCliDriver.java index def73d4..bf200b6 100644 --- itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkCliDriver.java +++ itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkCliDriver.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - + package org.apache.hadoop.hive.cli; import java.io.File; @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hadoop.hive.cli.control.SplitSupport; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -34,11 +35,13 @@ @RunWith(Parameterized.class) public class TestSparkCliDriver { + private static final int N_SPLITS = 11; + static CliAdapter adapter = new CliConfigs.SparkCliConfig().getCliAdapter(); @Parameters(name = "{0}") public static List getParameters() throws Exception { - return adapter.getParameters(); + return SplitSupport.process(adapter.getParameters(), TestSparkCliDriver.class, N_SPLITS); } @ClassRule diff --git itests/qtest/pom.xml itests/qtest/pom.xml index 1cc562b..ffebe09 100644 --- itests/qtest/pom.xml +++ itests/qtest/pom.xml @@ -426,15 +426,62 @@ test - - - - minimr - - - - - + + + + minimr + + + + + + qsplits + + + + org.apache.maven.plugins + maven-antrun-plugin + + + generate-split-tests + generate-sources + + + + + + + + + + + run + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-test-source + generate-sources + + add-test-source + + + + target/generated-test-sources + + + + + + + + + @@ -539,5 +586,4 @@ - diff --git itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestCliDriver.java itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestCliDriver.java index 4617d31..fed7a2a 100644 --- itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestCliDriver.java +++ itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestCliDriver.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hadoop.hive.cli.control.SplitSupport; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -33,11 +34,13 @@ @RunWith(Parameterized.class) public class TestCliDriver { + private static final int N_SPLITS = 35; + static CliAdapter adapter = new CliConfigs.CliConfig().getCliAdapter(); @Parameters(name = "{0}") public static List getParameters() throws Exception { - return adapter.getParameters(); + return SplitSupport.process(adapter.getParameters(), TestCliDriver.class, N_SPLITS); } @ClassRule diff --git itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestEncryptedHDFSCliDriver.java itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestEncryptedHDFSCliDriver.java index 76fce63..8b6efdb 100644 --- itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestEncryptedHDFSCliDriver.java +++ itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestEncryptedHDFSCliDriver.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hadoop.hive.cli.control.SplitSupport; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -33,11 +34,13 @@ @RunWith(Parameterized.class) public class TestEncryptedHDFSCliDriver { + private static final int N_SPLITS = 3; + static CliAdapter adapter = new CliConfigs.EncryptedHDFSCliConfig().getCliAdapter(); @Parameters(name = "{0}") public static List getParameters() throws Exception { - return adapter.getParameters(); + return SplitSupport.process(adapter.getParameters(), TestEncryptedHDFSCliDriver.class, N_SPLITS); } @ClassRule diff --git itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapCliDriver.java itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapCliDriver.java index cc05c11..c0dd2bf 100644 --- itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapCliDriver.java +++ itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapCliDriver.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hadoop.hive.cli.control.SplitSupport; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -33,11 +34,13 @@ @RunWith(Parameterized.class) public class TestMiniLlapCliDriver { + private static final int N_SPLITS = 3; + static CliAdapter adapter = new CliConfigs.MiniLlapCliConfig().getCliAdapter(); @Parameters(name = "{0}") public static List getParameters() throws Exception { - return adapter.getParameters(); + return SplitSupport.process(adapter.getParameters(), TestMiniLlapCliDriver.class, N_SPLITS); } @ClassRule diff --git itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java index 66d7bda..d068195 100644 --- itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java +++ itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hadoop.hive.cli.control.SplitSupport; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -35,9 +36,11 @@ static CliAdapter adapter = new CliConfigs.MiniLlapLocalCliConfig().getCliAdapter(); + private static int N_SPLITS = 18; + @Parameters(name = "{0}") public static List getParameters() throws Exception { - return adapter.getParameters(); + return SplitSupport.process(adapter.getParameters(), TestMiniLlapLocalCliDriver.class, N_SPLITS); } @ClassRule diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/SplitSupport.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/SplitSupport.java new file mode 100644 index 0000000..a51c5e4 --- /dev/null +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/SplitSupport.java @@ -0,0 +1,69 @@ +package org.apache.hadoop.hive.cli.control; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.google.common.annotations.VisibleForTesting; + +public class SplitSupport { + + public static List process(List parameters, Class currentClass, int nSplits) { + if (!isSplitExecution(currentClass)) { + return parameters; + } + // auto-disable primary test in case splits are present + if (isSplit0ClassExistsFor(currentClass)) { + return new ArrayList<>(); + } + int i = getSplitIndex(currentClass); + return getSplitParams(parameters, i, nSplits); + } + + private static boolean isSplitExecution(Class currentClass) { + return isSplitClass(currentClass) || isSplit0ClassExistsFor(currentClass); + } + + @VisibleForTesting + static List getSplitParams(List parameters, int i, int nSplits) { + if(i<0 || i>=nSplits) { + throw new IllegalArgumentException("unexpected"); + } + int n = parameters.size(); + int st = i * n / nSplits; + int ed = (i + 1) * n / nSplits; + + return parameters.subList(st, ed); + } + + @VisibleForTesting + static boolean isSplitClass(Class currentClass) { + Package p = currentClass.getPackage(); + return p.getName().matches(".*split[0-9]+$"); + } + + @VisibleForTesting + static int getSplitIndex(Class currentClass) { + Package p = currentClass.getPackage(); + Pattern pat = Pattern.compile("(.*split)([0-9]+)$"); + Matcher matcher = pat.matcher(p.getName()); + if (matcher.find()) { + return Integer.parseInt(matcher.group(2)); + } + throw new IllegalArgumentException("cant get splitindex for: " + p); + } + + @VisibleForTesting + static boolean isSplit0ClassExistsFor(Class clazz) { + Package p = clazz.getPackage(); + String split1 = p.getName() + ".split0." + clazz.getSimpleName(); + try { + Class c = Class.forName(split1); + return c != null; + } catch (ClassNotFoundException e) { + return false; + } + } + +} diff --git itests/util/src/test/java/org/apache/hadoop/hive/cli/control/TestSplitSupport.java itests/util/src/test/java/org/apache/hadoop/hive/cli/control/TestSplitSupport.java new file mode 100644 index 0000000..bb6cfca --- /dev/null +++ itests/util/src/test/java/org/apache/hadoop/hive/cli/control/TestSplitSupport.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.cli.control; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.junit.Test; + +import com.google.common.collect.Sets; + +public class TestSplitSupport { + + @Test + public void testSplitParams() { + + List input=new ArrayList<>(); + Set output=Sets.newIdentityHashSet(); + + } + + @Test + public void testIsSplitClass1() { + Class mainClass = org.apache.hadoop.hive.cli.control.splitsupport.SplitSupportDummy.class; + Class split0Class = + org.apache.hadoop.hive.cli.control.splitsupport.split0.SplitSupportDummy.class; + assertFalse(SplitSupport.isSplitClass(mainClass)); + assertTrue(SplitSupport.isSplitClass(split0Class)); + } + + @Test + public void testGetSplitIndex0() { + Class split0Class = org.apache.hadoop.hive.cli.control.splitsupport.split0.SplitSupportDummy.class; + assertEquals(0, SplitSupport.getSplitIndex(split0Class)); + } + + @Test + public void testGetSplitIndex125() { + Class split0Class = org.apache.hadoop.hive.cli.control.splitsupport.split125.SplitSupportDummy.class; + assertEquals(125, SplitSupport.getSplitIndex(split0Class)); + } +} diff --git itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/SplitSupportDummy.java itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/SplitSupportDummy.java new file mode 100644 index 0000000..335a50e --- /dev/null +++ itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/SplitSupportDummy.java @@ -0,0 +1,5 @@ +package org.apache.hadoop.hive.cli.control.splitsupport; + +public class SplitSupportDummy { + +} diff --git itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split0/SplitSupportDummy.java itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split0/SplitSupportDummy.java new file mode 100644 index 0000000..4ea134d --- /dev/null +++ itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split0/SplitSupportDummy.java @@ -0,0 +1,5 @@ +package org.apache.hadoop.hive.cli.control.splitsupport.split0; + +public class SplitSupportDummy { + +} diff --git itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split125/SplitSupportDummy.java itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split125/SplitSupportDummy.java new file mode 100644 index 0000000..254f8aa --- /dev/null +++ itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split125/SplitSupportDummy.java @@ -0,0 +1,5 @@ +package org.apache.hadoop.hive.cli.control.splitsupport.split125; + +public class SplitSupportDummy { + +}