diff --git data/files/datasets/testdataset/load.hive.sql data/files/datasets/testdataset/load.hive.sql new file mode 100644 index 0000000000..3436cda2df --- /dev/null +++ data/files/datasets/testdataset/load.hive.sql @@ -0,0 +1,10 @@ +-- +-- Table testdataset +-- +CREATE TABLE testdataset (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" INTO TABLE testdataset; + +ANALYZE TABLE testdataset COMPUTE STATISTICS; + +ANALYZE TABLE testdataset COMPUTE STATISTICS FOR COLUMNS key,value; diff --git data/scripts/q_test_init_src.sql data/scripts/q_test_init_src.sql index 2a62d29e34..765d6c7269 100644 --- data/scripts/q_test_init_src.sql +++ data/scripts/q_test_init_src.sql @@ -7,4 +7,3 @@ LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" OVERWRITE INTO TABLE ANALYZE TABLE src COMPUTE STATISTICS; ANALYZE TABLE src COMPUTE STATISTICS FOR COLUMNS key,value; - diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java index d38810f728..40c03304df 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java @@ -71,7 +71,7 @@ private Class cliAdapter; public AbstractCliConfig(Class adapter) { - cliAdapter=adapter; + cliAdapter = adapter; clusterType = MiniClusterType.none; queryFile = getSysPropValue("qfile"); queryFileRegex = getSysPropValue("qfile_regex"); @@ -135,7 +135,6 @@ protected void excludeQuery(String qFile) { excludedQueryFileNames.add(qFile); } - private static final Splitter TEST_SPLITTER = Splitter.onPattern("[, ]").trimResults().omitEmptyStrings(); diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java index eda6b692e7..b1d66a56b7 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java @@ -21,6 +21,8 @@ import static org.junit.Assert.fail; import com.google.common.base.Strings; + +import java.io.File; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Map; @@ -135,7 +137,7 @@ protected void runTestHelper(String tname, String fname, String fpath, boolean e System.err.println("Test " + fname + " skipped"); return; } - qt.cliInit(fname, false); + qt.cliInit(new File(fpath), false); int ecode = qt.executeClient(fname); if ((ecode == 0) ^ expectSuccess) { qt.failed(ecode, fname, debugHint); diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliAdapter.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliAdapter.java index 1fd295f5f6..574a67f2e3 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliAdapter.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliAdapter.java @@ -48,18 +48,18 @@ public CliAdapter(AbstractCliConfig cliConfig) { return ret; } - abstract public void beforeClass() throws Exception; + public abstract void beforeClass() throws Exception; // HIVE-14444 pending rename: before - abstract public void setUp(); + public abstract void setUp(); // HIVE-14444 pending rename: after - abstract public void tearDown(); + public abstract void tearDown(); // HIVE-14444 pending rename: afterClass - abstract public void shutdown() throws Exception; + public abstract void shutdown() throws Exception; - abstract public void runTest(String name, String name2, String absolutePath) throws Exception; + public abstract void runTest(String name, String fileName, String absolutePath) throws Exception; public final TestRule buildClassRule() { return new TestRule() { diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java index bd133f99df..d21fcaf0f8 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java @@ -19,6 +19,8 @@ import static org.junit.Assert.assertTrue; +import java.io.File; + import org.apache.hadoop.hive.accumulo.AccumuloQTestUtil; import org.apache.hadoop.hive.accumulo.AccumuloTestSetup; import org.apache.hadoop.hive.ql.QTestProcessExecResult; @@ -87,7 +89,7 @@ public void runTest(String tname, String fname, String fpath) throws Exception { return; } - qt.cliInit(fname); + qt.cliInit(new File(fpath)); qt.clearTestSideEffects(); int ecode = qt.executeClient(fname); if (ecode != 0) { diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java index 6c37391212..a7ec4f3d25 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java @@ -20,11 +20,9 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.File; import java.util.concurrent.TimeUnit; -import com.google.common.base.Stopwatch; -import com.google.common.base.Strings; -import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType; import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; @@ -36,6 +34,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Stopwatch; +import com.google.common.base.Strings; + public class CoreCliDriver extends CliAdapter { private static final Logger LOG = LoggerFactory.getLogger(CoreCliDriver.class); @@ -55,6 +56,7 @@ public void beforeClass() { final String hiveConfDir = cliConfig.getHiveConfDir(); final String initScript = cliConfig.getInitScript(); final String cleanupScript = cliConfig.getCleanupScript(); + try { final String hadoopVer = cliConfig.getHadoopVersion(); @@ -148,11 +150,13 @@ public Void invokeInternal() throws Exception { } } - static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " - + "or check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific test cases logs."; + private static String debugHint = + "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, " + + "or check ./ql/target/surefire-reports " + + "or ./itests/qtest/target/surefire-reports/ for specific test cases logs."; @Override - public void runTest(String tname, String fname, String fpath) throws Exception { + public void runTest(String testName, String fname, String fpath) throws Exception { Stopwatch sw = Stopwatch.createStarted(); boolean skipped = false; boolean failed = false; @@ -169,7 +173,7 @@ public void runTest(String tname, String fname, String fpath) throws Exception { return; } - qt.cliInit(fname, false); + qt.cliInit(new File(fpath), false); int ecode = qt.executeClient(fname); if (ecode != 0) { failed = true; @@ -178,8 +182,8 @@ public void runTest(String tname, String fname, String fpath) throws Exception { QTestProcessExecResult result = qt.checkCliDriverResults(fname); if (result.getReturnCode() != 0) { failed = true; - String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? - debugHint : "\r\n" + result.getCapturedOutput(); + String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? debugHint + : "\r\n" + result.getCapturedOutput(); qt.failedDiff(result.getReturnCode(), fname, message); } } @@ -187,7 +191,7 @@ public void runTest(String tname, String fname, String fpath) throws Exception { failed = true; qt.failed(e, fname, debugHint); } finally { - String message = "Done query" + fname + ". succeeded=" + !failed + ", skipped=" + skipped + + String message = "Done query " + fname + ". succeeded=" + !failed + ", skipped=" + skipped + ". ElapsedTime(ms)=" + sw.stop().elapsed(TimeUnit.MILLISECONDS); LOG.info(message); System.err.println(message); diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java index 3cc55a9d5c..fbca4241d4 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java @@ -137,7 +137,7 @@ public void runTest(String tname, String fname, String fpath) throws Exception { for (String versionFile : versionFiles) { // 1 for "_" after tname; 3 for ".qv" at the end. Version is in between. String versionStr = versionFile.substring(tname.length() + 1, versionFile.length() - 3); - outputs.add(qt.cliInit(tname + "." + versionStr, false)); + outputs.add(qt.cliInit(new File(queryDirectory, tname + "." + versionStr), false)); // TODO: will this work? ecode = qt.executeClient(versionFile, fname); if (ecode != 0) { diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java index c38cb2371b..24bdc58cab 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java @@ -21,6 +21,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.File; + import org.apache.hadoop.hive.hbase.HBaseQTestUtil; import org.apache.hadoop.hive.hbase.HBaseTestSetup; import org.apache.hadoop.hive.ql.QTestProcessExecResult; @@ -114,7 +116,7 @@ public void runTest(String tname, String fname, String fpath) throws Exception { return; } - qt.cliInit(fname, false); + qt.cliInit(new File(fpath), false); int ecode = qt.executeClient(fname); if (ecode != 0) { diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java index c171278e61..e828dc7b29 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java @@ -21,6 +21,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.File; + import org.apache.hadoop.hive.hbase.HBaseQTestUtil; import org.apache.hadoop.hive.hbase.HBaseTestSetup; import org.apache.hadoop.hive.ql.QTestProcessExecResult; @@ -95,7 +97,7 @@ public void runTest(String tname, String fname, String fpath) throws Exception { return; } - qt.cliInit(fname); + qt.cliInit(new File(fpath)); qt.clearTestSideEffects(); int ecode = qt.executeClient(fname); if (ecode == 0) { diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java index 847687797f..0788b2d4cb 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java @@ -20,6 +20,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.File; + import com.google.common.base.Strings; import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; @@ -119,7 +121,7 @@ public void runTest(String tname, String fname, String fpath) throws Exception { return; } - qt.cliInit(fname, false); + qt.cliInit(new File(fpath), false); int ecode = qt.executeClient(fname); if (ecode == 0) { qt.failed(fname, debugHint); diff --git itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java index 1a4c7df512..8ef9e0a280 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java +++ itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java @@ -23,6 +23,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.File; + import com.google.common.base.Strings; import org.apache.hadoop.hive.ql.QTestProcessExecResult; import org.apache.hadoop.hive.ql.QTestUtil; @@ -61,7 +63,7 @@ public void beforeClass() { String hadoopVer = cliConfig.getHadoopVersion(); qt = new QTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR, hiveConfDir, hadoopVer, initScript, - cleanupScript, false); + cleanupScript, false, null); // do a one time initialization qt.cleanUp(); @@ -126,7 +128,7 @@ public void runTest(String name, String fname, String fpath) throws Exception { return; } - qt.cliInit(fname, false); + qt.cliInit(new File(fpath), false); int ecode = qt.executeClient(fname); if (ecode != 0) { diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 6cd7a136ae..ed30f56c32 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -82,7 +82,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hive.cli.CliDriver; import org.apache.hadoop.hive.cli.CliSessionState; @@ -123,6 +122,9 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.processors.HiveCommand; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.dataset.DatasetCollection; +import org.apache.hadoop.hive.ql.dataset.DatasetParser; +import org.apache.hadoop.hive.ql.dataset.Dataset; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hive.common.util.StreamPrinter; @@ -165,8 +167,11 @@ public static final String PATH_HDFS_REGEX = "(hdfs://)([a-zA-Z0-9:/_\\-\\.=])+"; public static final String PATH_HDFS_WITH_DATE_USER_GROUP_REGEX = "([a-z]+) ([a-z]+)([ ]+)([0-9]+) ([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}) " + PATH_HDFS_REGEX; + public static final String TEST_SRC_TABLES_PROPERTY = "test.src.tables"; + private String testWarehouse; private final String testFiles; + private final File datasetDir; protected final String outDir; protected String overrideResultsDir; protected final String logDir; @@ -179,7 +184,7 @@ private final Set qNoSessionReuseQuerySet; private final Set qJavaVersionSpecificOutput; private static final String SORT_SUFFIX = ".sorted"; - private final Set srcTables; + private static Set srcTables; private final Set srcUDFs; private final MiniClusterType clusterType; private final FsType fsType; @@ -210,16 +215,40 @@ public interface SuiteAddTestFunctor { public void addTestToSuite(TestSuite suite, Object setup, String tName); } - private HBaseTestingUtility utility; public static Set getSrcTables() { - HashSet srcTables = new HashSet(); + if (srcTables == null){ + initSrcTables(); + } + return srcTables; + } + + public static void addSrcTable(String table){ + getSrcTables().add(table); + storeSrcTables(); + } + + public static Set initSrcTables() { + if (srcTables == null){ + initSrcTablesFromSystemProperty(); + storeSrcTables(); + } + + return srcTables; + } + + private static void storeSrcTables() { + System.setProperty(TEST_SRC_TABLES_PROPERTY, String.join(",", srcTables)); + } + + private static void initSrcTablesFromSystemProperty(){ + srcTables = new HashSet(); // FIXME: moved default value to here...for now // i think this features is never really used from the command line String defaultTestSrcTables = "src,src1,srcbucket,srcbucket2,src_json,src_thrift," + "src_sequencefile,srcpart,alltypesorc,src_hbase,cbo_t1,cbo_t2,cbo_t3,src_cbo,part," + "lineitem,alltypesparquet"; - for (String srcTable : System.getProperty("test.src.tables", defaultTestSrcTables).trim().split(",")) { + for (String srcTable : System.getProperty(TEST_SRC_TABLES_PROPERTY, defaultTestSrcTables).trim().split(",")) { srcTable = srcTable.trim(); if (!srcTable.isEmpty()) { srcTables.add(srcTable); @@ -228,9 +257,16 @@ if (srcTables.isEmpty()) { throw new RuntimeException("Source tables cannot be empty"); } - return srcTables; } + private CliDriver getCliDriver() { + if(cliDriver == null){ + cliDriver = new CliDriver(); + } + return cliDriver; + } + + /** * Returns the default UDF names which should not be removed when resetting the test database * @return The list of the UDF names not to remove @@ -253,8 +289,6 @@ return srcUDFs; } - - public HiveConf getConf() { return conf; } @@ -553,7 +587,6 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, } this.outDir = outDir; this.logDir = logDir; - this.srcTables=getSrcTables(); this.srcUDFs = getSrcUDFs(); // HIVE-14443 move this fall-back logic to CliConfigs @@ -600,6 +633,11 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, } testFiles = dataDir; + // Use path relative to dataDir directory if it is not specified + datasetDir = conf.get("test.data.set.files") == null + ? new File(new File(dataDir).getAbsolutePath() + "/datasets") + : new File(conf.get("test.data.set.files")); + // Use the current directory if it is not specified String scriptsDir = conf.get("test.data.scripts"); if (scriptsDir == null) { @@ -950,7 +988,7 @@ public void clearTablesCreatedDuringTests() throws Exception { for (String dbName : db.getAllDatabases()) { SessionState.get().setCurrentDatabase(dbName); for (String tblName : db.getAllTables()) { - if (!DEFAULT_DATABASE_NAME.equals(dbName) || !srcTables.contains(tblName)) { + if (!DEFAULT_DATABASE_NAME.equals(dbName) || !getSrcTables().contains(tblName)) { Table tblObj = null; try { tblObj = db.getTable(tblName); @@ -1029,8 +1067,8 @@ public void cleanUp() throws Exception { cleanUp(null); } - public void cleanUp(String tname) throws Exception { - boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname); + public void cleanUp(String fileName) throws Exception { + boolean canReuseSession = (fileName == null) || !qNoSessionReuseQuerySet.contains(fileName); if(!isSessionStateStarted) { startSessionState(canReuseSession); } @@ -1042,23 +1080,7 @@ public void cleanUp(String tname) throws Exception { clearUDFsCreatedDuringTests(); clearKeysCreatedInTests(); - File cleanupFile = new File(cleanupScript); - if (cleanupFile.isFile()) { - String cleanupCommands = readEntireFileIntoString(cleanupFile); - LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands); - if(cliDriver == null) { - cliDriver = new CliDriver(); - } - SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true); - int result = cliDriver.processLine(cleanupCommands); - if (result != 0) { - LOG.error("Failed during cleanup processLine with code={}. Ignoring", result); - // TODO Convert this to an Assert.fail once HIVE-14682 is fixed - } - SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", false); - } else { - LOG.info("No cleanup script detected. Skipping."); - } + cleanupFromFile(); // delete any contents in the warehouse dir Path p = new Path(testWarehouse); @@ -1079,6 +1101,22 @@ public void cleanUp(String tname) throws Exception { FunctionRegistry.unregisterTemporaryUDF("test_error"); } + private void cleanupFromFile() throws IOException { + File cleanupFile = new File(cleanupScript); + if (cleanupFile.isFile()) { + String cleanupCommands = readEntireFileIntoString(cleanupFile); + LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands); + + int result = getCliDriver().processLine(cleanupCommands); + if (result != 0) { + LOG.error("Failed during cleanup processLine with code={}. Ignoring", result); + // TODO Convert this to an Assert.fail once HIVE-14682 is fixed + } + } else { + LOG.info("No cleanup script detected. Skipping."); + } + } + protected void runCreateTableCmd(String createTableCmd) throws Exception { int ecode = 0; ecode = drv.run(createTableCmd).getResponseCode(); @@ -1105,33 +1143,68 @@ public void createSources() throws Exception { createSources(null); } - public void createSources(String tname) throws Exception { - boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname); + public void createSources(String fileName) throws Exception { + boolean canReuseSession = (fileName == null) || !qNoSessionReuseQuerySet.contains(fileName); if(!isSessionStateStarted) { startSessionState(canReuseSession); } - if(cliDriver == null) { - cliDriver = new CliDriver(); - } - cliDriver.processLine("set test.data.dir=" + testFiles + ";"); + getCliDriver().processLine("set test.data.dir=" + testFiles + ";"); + + conf.setBoolean("hive.test.init.phase", true); + + initFromScript(); + + conf.setBoolean("hive.test.init.phase", false); + } + + private void initFromScript() throws IOException { File scriptFile = new File(this.initScript); if (!scriptFile.isFile()) { LOG.info("No init script detected. Skipping"); return; } - conf.setBoolean("hive.test.init.phase", true); String initCommands = readEntireFileIntoString(scriptFile); LOG.info("Initial setup (" + initScript + "):\n" + initCommands); - int result = cliDriver.processLine(initCommands); + int result = getCliDriver().processLine(initCommands); LOG.info("Result from cliDrriver.processLine in createSources=" + result); if (result != 0) { Assert.fail("Failed during createSources processLine with code=" + result); } + } - conf.setBoolean("hive.test.init.phase", false); + private void initDataSetForTest(File file){ + DatasetParser parser = new DatasetParser(); + parser.parse(file); + + DatasetCollection datasets = parser.getDatasets(); + for (String table : datasets.getTables()){ + initDataset(table); + } + } + + private void initDataset(String table) { + if (getSrcTables().contains(table)){ + return; + } + + File tableFile = new File(new File(datasetDir, table), Dataset.INIT_FILE_NAME); + String commands = null; + try { + commands = readEntireFileIntoString(tableFile); + } catch (IOException e) { + throw new RuntimeException(String.format("dataset file not found %s", tableFile), e); + } + + int result = getCliDriver().processLine(commands); + LOG.info("Result from cliDrriver.processLine in initFromDatasets=" + result); + if (result != 0) { + Assert.fail("Failed during initFromDatasets processLine with code=" + result); + } + + addSrcTable(table); } public void init() throws Exception { @@ -1155,20 +1228,23 @@ public void init() throws Exception { sem = new SemanticAnalyzer(queryState); } - public void init(String tname) throws Exception { - cleanUp(tname); - createSources(tname); - cliDriver.processCmd("set hive.cli.print.header=true;"); + public void init(String fileName) throws Exception { + cleanUp(fileName); + createSources(fileName); + getCliDriver().processCmd("set hive.cli.print.header=true;"); } - public void cliInit(String tname) throws Exception { - cliInit(tname, true); + public void cliInit(File file) throws Exception { + cliInit(file, true); } - public String cliInit(String tname, boolean recreate) throws Exception { + public String cliInit(File file, boolean recreate) throws Exception { + String fileName = file.getName(); + initDataSetForTest(file); + if (recreate) { - cleanUp(tname); - createSources(tname); + cleanUp(fileName); + createSources(fileName); } HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER, @@ -1178,23 +1254,23 @@ public String cliInit(String tname, boolean recreate) throws Exception { assert ss != null; ss.in = System.in; - String outFileExtension = getOutFileExtension(tname); + String outFileExtension = getOutFileExtension(fileName); String stdoutName = null; if (outDir != null) { // TODO: why is this needed? - File qf = new File(outDir, tname); + File qf = new File(outDir, fileName); stdoutName = qf.getName().concat(outFileExtension); } else { - stdoutName = tname + outFileExtension; + stdoutName = fileName + outFileExtension; } File outf = new File(logDir, stdoutName); OutputStream fo = new BufferedOutputStream(new FileOutputStream(outf)); - if (qSortQuerySet.contains(tname)) { + if (qSortQuerySet.contains(fileName)) { ss.out = new SortPrintStream(fo, "UTF-8"); - } else if (qHashQuerySet.contains(tname)) { + } else if (qHashQuerySet.contains(fileName)) { ss.out = new DigestPrintStream(fo, "UTF-8"); - } else if (qSortNHashQuerySet.contains(tname)) { + } else if (qSortNHashQuerySet.contains(fileName)) { ss.out = new SortAndDigestPrintStream(fo, "UTF-8"); } else { ss.out = new PrintStream(fo, true, "UTF-8"); @@ -1203,7 +1279,7 @@ public String cliInit(String tname, boolean recreate) throws Exception { ss.setIsSilent(true); SessionState oldSs = SessionState.get(); - boolean canReuseSession = !qNoSessionReuseQuerySet.contains(tname); + boolean canReuseSession = !qNoSessionReuseQuerySet.contains(fileName); if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusterType.TEZ) { // Copy the tezSessionState from the old CliSessionState. TezSessionState tezSessionState = oldSs.getTezSession(); @@ -1229,10 +1305,10 @@ public String cliInit(String tname, boolean recreate) throws Exception { cliDriver = new CliDriver(); - if (tname.equals("init_file.q")) { + if (fileName.equals("init_file.q")) { ss.initFiles.add(AbstractCliConfig.HIVE_ROOT + "/data/scripts/test_init_file.sql"); } - cliDriver.processInitFiles(ss); + getCliDriver().processInitFiles(ss); return outf.getAbsolutePath(); } @@ -1288,7 +1364,7 @@ public int executeAdhocCommand(String q) { String q1 = q.split(";")[0] + ";"; LOG.debug("Executing " + q1); - return cliDriver.processLine(q1); + return getCliDriver().processLine(q1); } public int executeOne(String tname) { @@ -1303,7 +1379,7 @@ public int executeOne(String tname) { qMap.put(tname, qrest); System.out.println("Executing " + q1); - return cliDriver.processLine(q1); + return getCliDriver().processLine(q1); } public int execute(String tname) { @@ -1315,8 +1391,8 @@ public int executeClient(String tname1, String tname2) { return executeClientInternal(commands); } - public int executeClient(String tname) { - return executeClientInternal(getCommand(tname)); + public int executeClient(String fileName) { + return executeClientInternal(getCommand(fileName)); } private int executeClientInternal(String commands) { @@ -1342,7 +1418,7 @@ private int executeClientInternal(String commands) { if (isCommandUsedForTesting(command)) { rc = executeTestCommand(command); } else { - rc = cliDriver.processLine(command); + rc = getCliDriver().processLine(command); } if (rc != 0 && !ignoreErrors()) { @@ -1443,8 +1519,8 @@ private boolean isCommandUsedForTesting(final String command) { return testCommand != null; } - private String getCommand(String tname) { - String commands = qMap.get(tname); + private String getCommand(String fileName) { + String commands = qMap.get(fileName); StringBuilder newCommands = new StringBuilder(commands.length()); int lastMatchEnd = 0; Matcher commentMatcher = Pattern.compile("^--.*$", Pattern.MULTILINE).matcher(commands); @@ -1502,6 +1578,7 @@ public QTestProcessExecResult checkNegativeResults(String tname, Exception e) th } else if (e instanceof SemanticException) { outfd.write("Semantic Exception: \n"); } else { + outfd.close(); throw e; } @@ -1742,6 +1819,7 @@ private void maskPatterns(Pattern[] patterns, String fname) throws Exception { partialPlanMask = ppm.toArray(new PatternReplacementPair[ppm.size()]); } /* This list may be modified by specific cli drivers to mask strings that change on every test */ + @SuppressWarnings("serial") private final List> patternsWithMaskComments = new ArrayList>() { { @@ -2024,16 +2102,16 @@ public void tearDown() throws Exception { } /** - * QTRunner: Runnable class for running a a single query file. + * QTRunner: Runnable class for running a single query file. * **/ public static class QTRunner implements Runnable { private final QTestUtil qt; - private final String fname; + private final File file; - public QTRunner(QTestUtil qt, String fname) { + public QTRunner(QTestUtil qt, File file) { this.qt = qt; - this.fname = fname; + this.file = file; } @Override @@ -2041,10 +2119,10 @@ public void run() { try { // assumption is that environment has already been cleaned once globally // hence each thread does not call cleanUp() and createSources() again - qt.cliInit(fname, false); - qt.executeClient(fname); + qt.cliInit(file, false); + qt.executeClient(file.getName()); } catch (Throwable e) { - System.err.println("Query file " + fname + " failed with exception " + System.err.println("Query file " + file.getName() + " failed with exception " + e.getMessage()); e.printStackTrace(); outputTestFailureHelpMessage(); @@ -2097,7 +2175,7 @@ public static boolean queryListRunnerSingleThreaded(File[] qfiles, QTestUtil[] q qt[0].createSources(); for (int i = 0; i < qfiles.length && !failed; i++) { qt[i].clearTestSideEffects(); - qt[i].cliInit(qfiles[i].getName(), false); + qt[i].cliInit(qfiles[i], false); qt[i].executeClient(qfiles[i].getName()); QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName()); if (result.getReturnCode() != 0) { @@ -2147,7 +2225,7 @@ public static boolean queryListRunnerMultiThreaded(File[] qfiles, QTestUtil[] qt Thread[] qtThread = new Thread[qfiles.length]; for (int i = 0; i < qfiles.length; i++) { - qtRunners[i] = new QTRunner(qt[i], qfiles[i].getName()); + qtRunners[i] = new QTRunner(qt[i], qfiles[i]); qtThread[i] = new Thread(qtRunners[i]); } diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/Dataset.java itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/Dataset.java new file mode 100644 index 0000000000..0729f7603e --- /dev/null +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/Dataset.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.dataset; + +/** + * Dataset: simple class representation of a dataset + */ +public class Dataset { + public static final String INIT_FILE_NAME = "load.hive.sql"; + public static final String CLEANUP_FILE_NAME = "cleanup.hive.sql"; + + private String table; + + public Dataset(String table) { + this.table = table; + } + + public String getTable(){ + return table; + } +} diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/DatasetCollection.java itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/DatasetCollection.java new file mode 100644 index 0000000000..cf3ebd1103 --- /dev/null +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/DatasetCollection.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.dataset; + +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * DatasetCollection: utility wrapper class for a set of datasets + */ +public class DatasetCollection { + private Set coll = new HashSet(); + + public void add(Dataset dataset) { + coll.add(dataset); + } + + public void add(String table) { + add(new Dataset(table)); + } + + public Set getDatasets() { + return coll; + } + + public Set getTables() { + return coll.stream().map(d -> d.getTable()).collect(Collectors.toSet()); + } +} diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/DatasetParser.java itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/DatasetParser.java new file mode 100644 index 0000000000..ea0b0c532f --- /dev/null +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/dataset/DatasetParser.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.dataset; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * DatasetParser: a parser which could parse dataset "hooks" from q files, --!qt:dataset:mydataset + */ +public class DatasetParser { + + private DatasetCollection datasets = new DatasetCollection(); + private static final Logger LOG = LoggerFactory.getLogger("DatasetParser"); + + public static final String DATASET_PREFIX = "--! qt:dataset:"; + + public void parse(File file) { + try (BufferedReader br = new BufferedReader(new FileReader(file))) { + for (String line = br.readLine(); line != null; line = br.readLine()) { + if (line.trim().startsWith(DATASET_PREFIX)) { + Set strDatasets = parseDatasetsFromLine(line); + + for (String strDataset : strDatasets) { + datasets.add(strDataset); + } + } + } + } catch (IOException e) { + LOG.debug( + String.format("io exception while searching for datasets in qfile: %s", e.getMessage())); + } + } + + public DatasetCollection getDatasets() { + return datasets; + } + + public static Set parseDatasetsFromLine(String input) { + Set datasets = new HashSet(); + + input = input.substring(DATASET_PREFIX.length()); + if (!input.trim().isEmpty()) { + datasets.addAll(Arrays.asList(input.split(","))); + } + + return datasets; + } + +} diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java index 8cb3fad6a9..f087393d2b 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java @@ -22,7 +22,6 @@ import java.io.Serializable; import java.util.List; -import com.google.common.base.Strings; import org.apache.hadoop.hive.cli.control.AbstractCliConfig; import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; @@ -34,6 +33,8 @@ import org.junit.AfterClass; import org.junit.BeforeClass; +import com.google.common.base.Strings; + public class CoreParseNegative extends CliAdapter{ private static QTestUtil qt; diff --git itests/util/src/test/java/org/apache/hadoop/hive/ql/dataset/TestDatasetParser.java itests/util/src/test/java/org/apache/hadoop/hive/ql/dataset/TestDatasetParser.java new file mode 100644 index 0000000000..3668a67d8a --- /dev/null +++ itests/util/src/test/java/org/apache/hadoop/hive/ql/dataset/TestDatasetParser.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.dataset; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.junit.Assert; +import org.junit.Test; + +/** + * TestDatasetParser: test class for DataSetParser + */ +public class TestDatasetParser { + + @Test + public void testParseOne() { + Set expected = new HashSet(Arrays.asList("mydataset")); + Assert.assertEquals(expected, DatasetParser + .parseDatasetsFromLine(String.format("%smydataset", DatasetParser.DATASET_PREFIX))); + } + + @Test + public void testParseMultiple() { + Set expected = new HashSet(Arrays.asList("one", "two", "three")); + Assert.assertEquals(expected, DatasetParser + .parseDatasetsFromLine(String.format("%sone,two,three", DatasetParser.DATASET_PREFIX))); + } + + @Test + public void testParseUnique() { + Set expected = new HashSet(Arrays.asList("one", "two")); + Assert.assertEquals(expected, DatasetParser + .parseDatasetsFromLine(String.format("%sone,one,two", DatasetParser.DATASET_PREFIX))); + } + + @Test + public void testParseNone() { + Assert.assertTrue(DatasetParser + .parseDatasetsFromLine(String.format("%s", DatasetParser.DATASET_PREFIX)).isEmpty()); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java index cd4c61a971..47fff8b33e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java @@ -20,7 +20,8 @@ import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME; -import java.util.HashSet; +import java.util.Arrays; +import java.util.List; import java.util.Set; import org.apache.hadoop.hive.ql.metadata.Table; @@ -33,20 +34,6 @@ */ public class EnforceReadOnlyTables implements ExecuteWithHookContext { - private static final Set READ_ONLY_TABLES = new HashSet(); - - static { - for (String srcTable : System.getProperty("test.src.tables", "").trim().split(",")) { - srcTable = srcTable.trim(); - if (!srcTable.isEmpty()) { - READ_ONLY_TABLES.add(srcTable); - } - } - if (READ_ONLY_TABLES.isEmpty()) { - throw new AssertionError("Source tables cannot be empty"); - } - } - @Override public void run(HookContext hookContext) throws Exception { SessionState ss = SessionState.get(); @@ -65,12 +52,14 @@ public void run(SessionState sess, Set inputs, sess.getConf().getBoolean("hive.test.shutdown.phase", false)) { return; } + List readOnlyTables = Arrays.asList(System.getProperty("test.src.tables").split(",")); + for (WriteEntity w: outputs) { if ((w.getTyp() == WriteEntity.Type.TABLE) || (w.getTyp() == WriteEntity.Type.PARTITION)) { Table t = w.getTable(); if (DEFAULT_DATABASE_NAME.equalsIgnoreCase(t.getDbName()) - && READ_ONLY_TABLES.contains(t.getTableName())) { + && readOnlyTables.contains(t.getTableName())) { throw new RuntimeException ("Cannot overwrite read-only table: " + t.getTableName()); } } diff --git ql/src/test/queries/clientpositive/testdataset.q ql/src/test/queries/clientpositive/testdataset.q new file mode 100644 index 0000000000..96b78adfd0 --- /dev/null +++ ql/src/test/queries/clientpositive/testdataset.q @@ -0,0 +1,2 @@ +--! qt:dataset:testdataset +select count(*) from testdataset; diff --git ql/src/test/queries/clientpositive/testdataset_2.q ql/src/test/queries/clientpositive/testdataset_2.q new file mode 100644 index 0000000000..96b78adfd0 --- /dev/null +++ ql/src/test/queries/clientpositive/testdataset_2.q @@ -0,0 +1,2 @@ +--! qt:dataset:testdataset +select count(*) from testdataset; diff --git ql/src/test/results/clientpositive/testdataset.q.out ql/src/test/results/clientpositive/testdataset.q.out new file mode 100644 index 0000000000..2482befdb3 --- /dev/null +++ ql/src/test/results/clientpositive/testdataset.q.out @@ -0,0 +1,9 @@ +PREHOOK: query: select count(*) from testdataset +PREHOOK: type: QUERY +PREHOOK: Input: default@testdataset +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from testdataset +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testdataset +#### A masked pattern was here #### +500 diff --git ql/src/test/results/clientpositive/testdataset_2.q.out ql/src/test/results/clientpositive/testdataset_2.q.out new file mode 100644 index 0000000000..2482befdb3 --- /dev/null +++ ql/src/test/results/clientpositive/testdataset_2.q.out @@ -0,0 +1,9 @@ +PREHOOK: query: select count(*) from testdataset +PREHOOK: type: QUERY +PREHOOK: Input: default@testdataset +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from testdataset +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testdataset +#### A masked pattern was here #### +500