diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java index 7151372412..142f903ecc 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java @@ -33,6 +33,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.FsType; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hive.testutils.HiveTestEnvSetup; @@ -214,7 +215,7 @@ public boolean accept(File filePath) { // dedup file list Set testFiles = new TreeSet<>(); - if (queryFile != null && !queryFile.equals("")) { + if (isQFileSpecified()) { // The user may have passed a list of files - comma separated for (String qFile : TEST_SPLITTER.split(queryFile)) { File qF; @@ -223,10 +224,10 @@ public boolean accept(File filePath) { } else { qF = new File(qFile); } - if (excludedQueryFileNames.contains(qFile)) { + if (excludedQueryFileNames.contains(qFile) && !isQFileSpecified()) { LOG.warn(qF.getAbsolutePath() + " is among the excluded query files for this driver." + " Please update CliConfigs.java or testconfiguration.properties file to" - + " include the qfile"); + + " include the qfile or specify qfile through command line explicitly: -Dqfile=test.q"); } testFiles.add(qF); } @@ -241,12 +242,20 @@ public boolean accept(File filePath) { } for (String qFileName : excludedQueryFileNames) { - testFiles.remove(new File(queryDir, qFileName)); + // in case of running as ptest, exclusions should be respected, + // because test drivers receive every qfiles regardless of exclusions + if ("hiveptest".equals(System.getProperty("user.name")) || !isQFileSpecified()) { + testFiles.remove(new File(queryDir, qFileName)); + } } return testFiles; } + public boolean isQFileSpecified() { + return queryFile != null && !queryFile.equals(""); + } + private void prepareDirs() throws Exception { File hiveRootDir = new File(HIVE_ROOT); if (!hiveRootDir.exists()) { diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java index a17c508624..254cc953b0 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java @@ -25,7 +25,9 @@ import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -52,6 +54,10 @@ public static final String MASK_PATTERN = "#### A masked pattern was here ####"; public static final String PARTIAL_MASK_PATTERN = "#### A PARTIAL masked pattern was here ####"; + private final Set qMaskStatsQuerySet = new HashSet(); + private final Set qMaskDataSizeQuerySet = new HashSet(); + private final Set qMaskLineageQuerySet = new HashSet(); + private static final PatternReplacementPair MASK_STATS = new PatternReplacementPair( Pattern.compile(" Num rows: [1-9][0-9]* Data size: [1-9][0-9]*"), " Num rows: ###Masked### Data size: ###Masked###"); @@ -62,6 +68,10 @@ Pattern.compile("POSTHOOK: Lineage: .*"), "POSTHOOK: Lineage: ###Masked###"); + private static final Pattern PATTERN_MASK_STATS = Pattern.compile("-- MASK_STATS"); + private static final Pattern PATTERN_MASK_DATA_SIZE = Pattern.compile("-- MASK_DATA_SIZE"); + private static final Pattern PATTERN_MASK_LINEAGE = Pattern.compile("-- MASK_LINEAGE"); + private FsType fsType = FsType.local; public static class LineProcessingResult { @@ -141,7 +151,8 @@ public QOutProcessor() { return patterns; } - public void maskPatterns(String fname, boolean maskStats, boolean maskDataSize, boolean maskLineage) throws Exception { + public void maskPatterns(String fname, String tname) throws Exception { + String line; BufferedReader in; BufferedWriter out; @@ -156,7 +167,7 @@ public void maskPatterns(String fname, boolean maskStats, boolean maskDataSize, boolean lastWasMasked = false; while (null != (line = in.readLine())) { - LineProcessingResult result = processLine(line, maskStats, maskDataSize, maskLineage); + LineProcessingResult result = processLine(line, tname); if (result.line.equals(MASK_PATTERN)) { // We're folding multiple masked lines into one. @@ -178,7 +189,7 @@ public void maskPatterns(String fname, boolean maskStats, boolean maskDataSize, out.close(); } - public LineProcessingResult processLine(String line, boolean maskStats, boolean maskDataSize, boolean maskLineage) { + public LineProcessingResult processLine(String line, String tname) { LineProcessingResult result = new LineProcessingResult(line); Matcher matcher = null; @@ -216,7 +227,7 @@ public LineProcessingResult processLine(String line, boolean maskStats, boolean } } - if (!result.partialMaskWasMatched && maskStats) { + if (!result.partialMaskWasMatched && qMaskStatsQuerySet.contains(tname)) { matcher = MASK_STATS.pattern.matcher(result.line); if (matcher.find()) { result.line = result.line.replaceAll(MASK_STATS.pattern.pattern(), MASK_STATS.replacement); @@ -224,7 +235,7 @@ public LineProcessingResult processLine(String line, boolean maskStats, boolean } } - if (!result.partialMaskWasMatched && maskDataSize) { + if (!result.partialMaskWasMatched && qMaskDataSizeQuerySet.contains(tname)) { matcher = MASK_DATA_SIZE.pattern.matcher(result.line); if (matcher.find()) { result.line = result.line.replaceAll(MASK_DATA_SIZE.pattern.pattern(), MASK_DATA_SIZE.replacement); @@ -232,7 +243,7 @@ public LineProcessingResult processLine(String line, boolean maskStats, boolean } } - if (!result.partialMaskWasMatched && maskLineage) { + if (!result.partialMaskWasMatched && qMaskLineageQuerySet.contains(tname)) { matcher = MASK_LINEAGE.pattern.matcher(result.line); if (matcher.find()) { result.line = result.line.replaceAll(MASK_LINEAGE.pattern.pattern(), MASK_LINEAGE.replacement); @@ -298,5 +309,25 @@ public LineProcessingResult processLine(String line, boolean maskStats, boolean public void addPatternWithMaskComment(String patternStr, String maskComment) { patternsWithMaskComments.add(toPatternPair(patternStr, maskComment)); } - + + public void initMasks(File qf, String query) { + if (matches(PATTERN_MASK_STATS, query)) { + qMaskStatsQuerySet.add(qf.getName()); + } + if (matches(PATTERN_MASK_DATA_SIZE, query)) { + qMaskDataSizeQuerySet.add(qf.getName()); + } + if (matches(PATTERN_MASK_LINEAGE, query)) { + qMaskLineageQuerySet.add(qf.getName()); + } + } + + private boolean matches(Pattern pattern, String query) { + Matcher matcher = pattern.matcher(query); + if (matcher.find()) { + return true; + } + return false; + } + } diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 59c7ac4c2c..dcf2d49f0f 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -156,6 +156,7 @@ private static final String BUILD_DIR_PROPERTY = "build.dir"; // typically target private static final String TEST_SRC_TABLES_PROPERTY = "test.src.tables"; + /** * The default Erasure Coding Policy to use in Erasure Coding tests. */ @@ -173,9 +174,6 @@ private final Set qHashQuerySet; private final Set qSortNHashQuerySet; private final Set qNoSessionReuseQuerySet; - private final Set qMaskStatsQuerySet; - private final Set qMaskDataSizeQuerySet; - private final Set qMaskLineageQuerySet; private static final String SORT_SUFFIX = ".sorted"; private static Set srcTables; private final Set srcUDFs; @@ -495,9 +493,6 @@ public QTestUtil(QTestArguments testArgs) throws Exception { qHashQuerySet = new HashSet(); qSortNHashQuerySet = new HashSet(); qNoSessionReuseQuerySet = new HashSet(); - qMaskStatsQuerySet = new HashSet(); - qMaskDataSizeQuerySet = new HashSet(); - qMaskLineageQuerySet = new HashSet(); this.clusterType = testArgs.getClusterType(); HadoopShims shims = ShimLoader.getHadoopShims(); @@ -534,12 +529,16 @@ public QTestUtil(QTestArguments testArgs) throws Exception { this.initScript = scriptsDir + File.separator + testArgs.getInitScript(); this.cleanupScript = scriptsDir + File.separator + testArgs.getCleanupScript(); - overWrite = "true".equalsIgnoreCase(System.getProperty("test.output.overwrite")); + overWrite = shouldOverwriteResults(); init(); savedConf = new HiveConf(conf); } + private boolean shouldOverwriteResults() { + return "true".equalsIgnoreCase(System.getProperty("test.output.overwrite")); + } + private String getScriptsDir() { // Use the current directory if it is not specified String scriptsDir = conf.get("test.data.scripts"); @@ -802,15 +801,7 @@ public void addFile(File qf, boolean partial) throws IOException { qNoSessionReuseQuerySet.add(qf.getName()); } - if (matches(MASK_STATS, query)) { - qMaskStatsQuerySet.add(qf.getName()); - } - if (matches(MASK_DATA_SIZE, query)) { - qMaskDataSizeQuerySet.add(qf.getName()); - } - if (matches(MASK_LINEAGE, query)) { - qMaskLineageQuerySet.add(qf.getName()); - } + qOutProcessor.initMasks(qf, query); } private static final Pattern SORT_BEFORE_DIFF = Pattern.compile("-- SORT_BEFORE_DIFF"); @@ -818,9 +809,6 @@ public void addFile(File qf, boolean partial) throws IOException { private static final Pattern HASH_QUERY_RESULTS = Pattern.compile("-- HASH_QUERY_RESULTS"); private static final Pattern SORT_AND_HASH_QUERY_RESULTS = Pattern.compile("-- SORT_AND_HASH_QUERY_RESULTS"); private static final Pattern NO_SESSION_REUSE = Pattern.compile("-- NO_SESSION_REUSE"); - private static final Pattern MASK_STATS = Pattern.compile("-- MASK_STATS"); - private static final Pattern MASK_DATA_SIZE = Pattern.compile("-- MASK_DATA_SIZE"); - private static final Pattern MASK_LINEAGE = Pattern.compile("-- MASK_LINEAGE"); private boolean matches(Pattern pattern, String query) { Matcher matcher = pattern.matcher(query); @@ -1532,8 +1520,7 @@ public QTestProcessExecResult checkCliDriverResults(String tname) throws Excepti File f = new File(logDir, tname + outFileExtension); - qOutProcessor.maskPatterns(f.getPath(), - qMaskStatsQuerySet.contains(tname), qMaskDataSizeQuerySet.contains(tname), qMaskLineageQuerySet.contains(tname)); + qOutProcessor.maskPatterns(f.getPath(), tname); QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(), outFileName, false, qSortSet.contains(tname)); @@ -1549,11 +1536,9 @@ public QTestProcessExecResult checkCliDriverResults(String tname) throws Excepti public QTestProcessExecResult checkCompareCliDriverResults(String tname, List outputs) throws Exception { assert outputs.size() > 1; - qOutProcessor.maskPatterns(outputs.get(0), - qMaskStatsQuerySet.contains(tname), qMaskDataSizeQuerySet.contains(tname), qMaskLineageQuerySet.contains(tname)); + qOutProcessor.maskPatterns(outputs.get(0), tname); for (int i = 1; i < outputs.size(); ++i) { - qOutProcessor.maskPatterns(outputs.get(i), - qMaskStatsQuerySet.contains(tname), qMaskDataSizeQuerySet.contains(tname), qMaskLineageQuerySet.contains(tname)); + qOutProcessor.maskPatterns(outputs.get(i), tname); QTestProcessExecResult result = executeDiffCommand( outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname)); if (result.getReturnCode() != 0) { diff --git itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java index c193da375c..9e19f27b7f 100644 --- itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java +++ itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java @@ -68,6 +68,6 @@ public void testSelectiveHdfsPatternMaskOnlyHdfsPath() { } private String processLine(String line) { - return qOutProcessor.processLine(line, false, false, false).get(); + return qOutProcessor.processLine(line, null).get(); } } \ No newline at end of file