diff --git a/data/scripts/q_test_cleanup_compare.sql b/data/scripts/q_test_cleanup_compare.sql new file mode 100644 index 0000000..0c6ab14 --- /dev/null +++ b/data/scripts/q_test_cleanup_compare.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS alltypesorc; diff --git a/data/scripts/q_test_cleanup_contrib.sql b/data/scripts/q_test_cleanup_contrib.sql new file mode 100644 index 0000000..95d4ba0 --- /dev/null +++ b/data/scripts/q_test_cleanup_contrib.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS src; +DROP TABLE IF EXISTS src_thrift; diff --git a/data/scripts/q_test_cleanup_for_encryption.sql b/data/scripts/q_test_cleanup_for_encryption.sql deleted file mode 100644 index 070cdbe..0000000 --- a/data/scripts/q_test_cleanup_for_encryption.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE IF EXISTS src PURGE; diff --git a/data/scripts/q_test_cleanup_src.sql b/data/scripts/q_test_cleanup_src.sql new file mode 100644 index 0000000..08264ee --- /dev/null +++ b/data/scripts/q_test_cleanup_src.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS src; diff --git a/data/scripts/q_test_cleanup_tez.sql b/data/scripts/q_test_cleanup_tez.sql new file mode 100644 index 0000000..89050ec --- /dev/null +++ b/data/scripts/q_test_cleanup_tez.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS src; +DROP TABLE IF EXISTS alltypesorc; diff --git a/data/scripts/q_test_init_compare.sql b/data/scripts/q_test_init_compare.sql new file mode 100644 index 0000000..c554250 --- /dev/null +++ b/data/scripts/q_test_init_compare.sql @@ -0,0 +1,26 @@ +set hive.stats.dbclass=fs; +-- +-- Table alltypesorc +-- +DROP TABLE IF EXISTS alltypesorc; +CREATE TABLE alltypesorc( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) + STORED AS ORC; + +LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/alltypesorc" +OVERWRITE INTO TABLE alltypesorc; + +ANALYZE TABLE alltypesorc COMPUTE STATISTICS; + +ANALYZE TABLE alltypesorc COMPUTE STATISTICS FOR COLUMNS ctinyint,csmallint,cint,cbigint,cfloat,cdouble,cstring1,cstring2,ctimestamp1,ctimestamp2,cboolean1,cboolean2; diff --git a/data/scripts/q_test_init_contrib.sql b/data/scripts/q_test_init_contrib.sql new file mode 100644 index 0000000..b7bb5ab --- /dev/null +++ b/data/scripts/q_test_init_contrib.sql @@ -0,0 +1,29 @@ +set hive.stats.dbclass=fs; +-- +-- Table src +-- +DROP TABLE IF EXISTS src; + +CREATE TABLE src (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" INTO TABLE src; + +ANALYZE TABLE src COMPUTE STATISTICS; + +ANALYZE TABLE src COMPUTE STATISTICS FOR COLUMNS key,value; + +-- +-- Table src_thrift +-- +DROP TABLE IF EXISTS src_thrift; + +CREATE TABLE src_thrift +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer' +WITH SERDEPROPERTIES ( + 'serialization.class' = 'org.apache.hadoop.hive.serde2.thrift.test.Complex', + 'serialization.format' = 'org.apache.thrift.protocol.TBinaryProtocol') +STORED AS SEQUENCEFILE; + +LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/complex.seq" INTO TABLE src_thrift; + +ANALYZE TABLE src_thrift COMPUTE STATISTICS; diff --git a/data/scripts/q_test_init_for_encryption.sql b/data/scripts/q_test_init_for_encryption.sql deleted file mode 100644 index 56b44e0..0000000 --- a/data/scripts/q_test_init_for_encryption.sql +++ /dev/null @@ -1,5 +0,0 @@ -DROP TABLE IF EXISTS src PURGE; - -CREATE TABLE src(key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE; - -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" OVERWRITE INTO TABLE src; diff --git a/data/scripts/q_test_init_src.sql b/data/scripts/q_test_init_src.sql new file mode 100644 index 0000000..56b44e0 --- /dev/null +++ b/data/scripts/q_test_init_src.sql @@ -0,0 +1,5 @@ +DROP TABLE IF EXISTS src PURGE; + +CREATE TABLE src(key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" OVERWRITE INTO TABLE src; diff --git a/data/scripts/q_test_init_src_with_stats.sql b/data/scripts/q_test_init_src_with_stats.sql new file mode 100644 index 0000000..765d6c7 --- /dev/null +++ b/data/scripts/q_test_init_src_with_stats.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS src PURGE; + +CREATE TABLE src(key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" OVERWRITE INTO TABLE src; + +ANALYZE TABLE src COMPUTE STATISTICS; + +ANALYZE TABLE src COMPUTE STATISTICS FOR COLUMNS key,value; diff --git a/data/scripts/q_test_init_tez.sql b/data/scripts/q_test_init_tez.sql new file mode 100644 index 0000000..9e3c37e --- /dev/null +++ b/data/scripts/q_test_init_tez.sql @@ -0,0 +1,35 @@ +DROP TABLE IF EXISTS src PURGE; + +CREATE TABLE src(key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" OVERWRITE INTO TABLE src; + +ANALYZE TABLE src COMPUTE STATISTICS; + +ANALYZE TABLE src COMPUTE STATISTICS FOR COLUMNS key,value; + +-- +-- Table alltypesorc +-- +DROP TABLE IF EXISTS alltypesorc; +CREATE TABLE alltypesorc( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) + STORED AS ORC; + +LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/alltypesorc" +OVERWRITE INTO TABLE alltypesorc; + +ANALYZE TABLE alltypesorc COMPUTE STATISTICS; + +ANALYZE TABLE alltypesorc COMPUTE STATISTICS FOR COLUMNS ctinyint,csmallint,cint,cbigint,cfloat,cdouble,cstring1,cstring2,ctimestamp1,ctimestamp2,cboolean1,cboolean2; diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java index 198fe48..ad2baa2 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java @@ -36,7 +36,8 @@ public void testMTQueries1() throws Exception { String[] testNames = new String[] {"join2.q", "groupby1.q", "input1.q", "input19.q"}; File[] qfiles = setupQFiles(testNames); - QTestUtil[] qts = QTestUtil.queryListRunnerSetup(qfiles, resDir, logDir); + QTestUtil[] qts = QTestUtil.queryListRunnerSetup(qfiles, resDir, logDir, "q_test_init_src_with_stats.sql", + "q_test_cleanup_src_with_stats.sql"); for (QTestUtil util : qts) { // derby fails creating multiple stats aggregator concurrently util.getConf().setBoolean("hive.exec.submitviachild", true); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java index 201c3d3..e7748b7 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java @@ -124,8 +124,8 @@ public MiniTezCliConfig() { setResultsDir("ql/src/test/results/clientpositive/tez"); setLogDir("itests/qtest/target/qfile-results/clientpositive"); - setInitScript("q_test_init.sql"); - setCleanupScript("q_test_cleanup.sql"); + setInitScript("q_test_init_tez.sql"); + setCleanupScript("q_test_cleanup_tez.sql"); setHiveConfDir("data/conf/tez"); setClusterType(MiniClusterType.tez); @@ -198,8 +198,8 @@ public EncryptedHDFSCliConfig() { setResultsDir("ql/src/test/results/clientpositive/encrypted"); setLogDir("itests/qtest/target/qfile-results/clientpositive"); - setInitScript("q_test_init_for_encryption.sql"); - setCleanupScript("q_test_cleanup_for_encryption.sql"); + setInitScript("q_test_init_src.sql"); + setCleanupScript("q_test_cleanup_src.sql"); setClusterType(MiniClusterType.mr); @@ -225,8 +225,8 @@ public ContribCliConfig() { setResultsDir("contrib/src/test/results/clientpositive"); setLogDir("itests/qtest/target/qfile-results/contribclientpositive"); - setInitScript("q_test_init.sql"); - setCleanupScript("q_test_cleanup.sql"); + setInitScript("q_test_init_contrib.sql"); + setCleanupScript("q_test_cleanup_contrib.sql"); setHiveConfDir(""); } catch (Exception e) { @@ -268,8 +268,8 @@ public CompareCliConfig() { setResultsDir("ql/src/test/results/clientcompare"); setLogDir("itests/qtest/target/qfile-results/clientcompare"); - setInitScript("q_test_init.sql"); - setCleanupScript("q_test_cleanup.sql"); + setInitScript("q_test_init_compare.sql"); + setCleanupScript("q_test_cleanup_compare.sql"); setHiveConfDir(""); setClusterType(MiniClusterType.none); @@ -313,8 +313,8 @@ public NegativeMinimrCli() { setResultsDir("ql/src/test/results/clientnegative"); setLogDir("itests/qtest/target/qfile-results/clientnegative"); - setInitScript("q_test_init_for_encryption.sql"); - setCleanupScript("q_test_cleanup_for_encryption.sql"); + setInitScript("q_test_init_src.sql"); + setCleanupScript("q_test_cleanup_src.sql"); setHiveConfDir(""); setClusterType(MiniClusterType.mr); @@ -333,8 +333,8 @@ public HBaseCliConfig() { setResultsDir("hbase-handler/src/test/results/positive"); setLogDir("itests/qtest/target/qfile-results/hbase-handler/positive"); - setInitScript("q_test_init.sql"); - setCleanupScript("q_test_cleanup.sql"); + setInitScript("q_test_init_src_with_stats.sql"); + setCleanupScript("q_test_cleanup_src.sql"); setHiveConfDir(""); setClusterType(MiniClusterType.none); @@ -353,8 +353,8 @@ public DummyConfig() { setResultsDir("ql/src/test/results/clientcompare"); setLogDir("itests/qtest/target/qfile-results/clientcompare"); - setInitScript("q_test_init.sql"); - setCleanupScript("q_test_cleanup.sql"); + setInitScript("q_test_init_compare.sql"); + setCleanupScript("q_test_cleanup_compare.sql"); setHiveConfDir(""); setClusterType(MiniClusterType.none); @@ -373,8 +373,8 @@ public HBaseNegativeCliConfig() { setResultsDir("hbase-handler/src/test/results/negative"); setLogDir("itests/qtest/target/qfile-results/hbase-handler/negative"); - setInitScript("q_test_init.sql"); - setCleanupScript("q_test_cleanup.sql"); + setInitScript("q_test_init_src.sql"); + setCleanupScript("q_test_cleanup_src.sql"); setHiveConfDir(""); setClusterType(MiniClusterType.none); @@ -439,8 +439,8 @@ public AccumuloCliConfig() { setResultsDir("accumulo-handler/src/test/results/positive"); setLogDir("itests/qtest/target/qfile-results/accumulo-handler/positive"); - setInitScript("q_test_init.sql"); - setCleanupScript("q_test_cleanup.sql"); + setInitScript("q_test_init_src_with_stats.sql"); + setCleanupScript("q_test_cleanup_src.sql"); setHiveConfDir(""); setClusterType(MiniClusterType.none); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index e49ecd9..5f683f0 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -1965,12 +1965,14 @@ public void run() { * @return one QTestUtil for each query file */ public static QTestUtil[] queryListRunnerSetup(File[] qfiles, String resDir, - String logDir) throws Exception + String logDir, String initScript, String cleanupScript) throws Exception { QTestUtil[] qt = new QTestUtil[qfiles.length]; for (int i = 0; i < qfiles.length; i++) { qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20", - defaultInitScript, defaultCleanupScript, false, false); + initScript == null ? defaultInitScript : initScript, + cleanupScript == null ? defaultCleanupScript : cleanupScript, + false, false); qt[i].addFile(qfiles[i]); qt[i].clearTestSideEffects(); }