diff --git data/scripts/q_test_cleanup.sql data/scripts/q_test_cleanup.sql index 31bd720..8ec0f9f 100644 --- data/scripts/q_test_cleanup.sql +++ data/scripts/q_test_cleanup.sql @@ -7,4 +7,12 @@ DROP TABLE IF EXISTS srcbucket; DROP TABLE IF EXISTS srcbucket2; DROP TABLE IF EXISTS srcpart; DROP TABLE IF EXISTS primitives; - +DROP TABLE IF EXISTS dest1; +DROP TABLE IF EXISTS dest2; +DROP TABLE IF EXISTS dest3; +DROP TABLE IF EXISTS dest4; +DROP TABLE IF EXISTS dest4_sequencefile; +DROP TABLE IF EXISTS dest_j1; +DROP TABLE IF EXISTS dest_g1; +DROP TABLE IF EXISTS dest_g2; +DROP TABLE IF EXISTS fetchtask_ioexception; diff --git data/scripts/q_test_init.sql data/scripts/q_test_init.sql index 12afdf3..9ffaf7f 100644 --- data/scripts/q_test_init.sql +++ data/scripts/q_test_init.sql @@ -5,7 +5,7 @@ DROP TABLE IF EXISTS src; CREATE TABLE src (key STRING, value STRING) STORED AS TEXTFILE; -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" INTO TABLE src; +LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" INTO TABLE src; -- -- Table src1 @@ -14,7 +14,7 @@ DROP TABLE IF EXISTS src1; CREATE TABLE src1 (key STRING, value STRING) STORED AS TEXTFILE; -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv3.txt" INTO TABLE src1; +LOAD DATA LOCAL INPATH "../../data/files/kv3.txt" INTO TABLE src1; -- -- Table src_json @@ -23,7 +23,7 @@ DROP TABLE IF EXISTS src_json; CREATE TABLE src_json (json STRING) STORED AS TEXTFILE; -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/json.txt" INTO TABLE src_json; +LOAD DATA LOCAL INPATH "../../data/files/json.txt" INTO TABLE src_json; -- @@ -33,7 +33,7 @@ DROP TABLE IF EXISTS src_sequencefile; CREATE TABLE src_sequencefile (key STRING, value STRING) STORED AS SEQUENCEFILE; -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.seq" INTO TABLE src_sequencefile; +LOAD DATA LOCAL INPATH "../../data/files/kv1.seq" INTO TABLE src_sequencefile; -- @@ -48,7 +48,7 @@ WITH SERDEPROPERTIES ( 'serialization.format' = 'com.facebook.thrift.protocol.TBinaryProtocol') STORED AS SEQUENCEFILE; -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/complex.seq" INTO TABLE src_thrift; +LOAD DATA LOCAL INPATH "../../data/files/complex.seq" INTO TABLE src_thrift; -- @@ -60,8 +60,8 @@ CREATE TABLE srcbucket (key INT, value STRING) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE; -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket0.txt" INTO TABLE srcbucket; -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket1.txt" INTO TABLE srcbucket; +LOAD DATA LOCAL INPATH "../../data/files/srcbucket0.txt" INTO TABLE srcbucket; +LOAD DATA LOCAL INPATH "../../data/files/srcbucket1.txt" INTO TABLE srcbucket; -- @@ -73,8 +73,8 @@ CREATE TABLE srcbucket2 (key INT, value STRING) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE; -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket20.txt" INTO TABLE srcbucket2; -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/srcbucket21.txt" INTO TABLE srcbucket2; +LOAD DATA LOCAL INPATH "../../data/files/srcbucket20.txt" INTO TABLE srcbucket2; +LOAD DATA LOCAL INPATH "../../data/files/srcbucket21.txt" INTO TABLE srcbucket2; -- @@ -86,19 +86,45 @@ CREATE TABLE srcpart (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) STORED AS TEXTFILE; -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" +LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" OVERWRITE INTO TABLE srcpart PARTITION (ds="2008-04-08", hr="11"); -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" +LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" OVERWRITE INTO TABLE srcpart PARTITION (ds="2008-04-08", hr="12"); -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" +LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" OVERWRITE INTO TABLE srcpart PARTITION (ds="2008-04-09", hr="11"); -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" +LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" OVERWRITE INTO TABLE srcpart PARTITION (ds="2008-04-09", hr="12"); +-- +-- Table alltypesorc +-- +DROP TABLE IF EXISTS alltypesorc; +CREATE TABLE alltypesorc( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) + STORED AS ORC; + +LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" +OVERWRITE INTO TABLE alltypesorc; + + +-- +-- Table primitives +-- DROP TABLE IF EXISTS primitives; CREATE TABLE primitives ( id INT, @@ -118,15 +144,17 @@ ROW FORMAT DELIMITED ESCAPED BY '\\' STORED AS TEXTFILE; -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/types/primitives/090101.txt" +LOAD DATA LOCAL INPATH "../../data/files/types/primitives/090101.txt" OVERWRITE INTO TABLE primitives PARTITION(year=2009, month=1); -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/types/primitives/090201.txt" +LOAD DATA LOCAL INPATH "../../data/files/types/primitives/090201.txt" OVERWRITE INTO TABLE primitives PARTITION(year=2009, month=2); -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/types/primitives/090301.txt" +LOAD DATA LOCAL INPATH "../../data/files/types/primitives/090301.txt" OVERWRITE INTO TABLE primitives PARTITION(year=2009, month=3); -LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/types/primitives/090401.txt" +LOAD DATA LOCAL INPATH "../../data/files/types/primitives/090401.txt" OVERWRITE INTO TABLE primitives PARTITION(year=2009, month=4); +DROP FUNCTION IF EXISTS qtest_get_java_boolean; +CREATE FUNCTION qtest_get_java_boolean AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaBoolean'; \ No newline at end of file diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 2fefa06..4b05482 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -38,7 +38,6 @@ import java.io.PrintStream; import java.io.Serializable; import java.io.StringWriter; -import java.io.UnsupportedEncodingException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; @@ -142,6 +141,7 @@ private boolean miniMr = false; private String hadoopVer = null; private QTestSetup setup = null; + private boolean isSessionStateStarted = false; static { for (String srcTable : System.getProperty("test.src.tables", "").trim().split(",")) { @@ -593,14 +593,16 @@ public void clearTestSideEffects() throws Exception { } public void cleanUp() throws Exception { - // Drop any tables that remain due to unsuccessful runs - for (String s : new String[] {"src", "src1", "src_json", "src_thrift", - "src_sequencefile", "srcpart", "srcbucket", "srcbucket2", "dest1", - "dest2", "dest3", "dest4", "dest4_sequencefile", "dest_j1", "dest_j2", - "dest_g1", "dest_g2", "fetchtask_ioexception", - AllVectorTypesRecord.TABLE_NAME}) { - db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, s); + if(!isSessionStateStarted) { + startSessionState(); } + final String cleanupScript = "../../data/scripts/q_test_cleanup.sql"; + String cleanupCommands = readEntireFileIntoString(new File(cleanupScript)); + LOG.info("Cleanup:\n" + cleanupCommands); + if(cliDriver == null) { + cliDriver = new CliDriver(); + } + cliDriver.processLine(cleanupCommands); // delete any contents in the warehouse dir Path p = new Path(testWarehouse); @@ -653,119 +655,20 @@ private void runCmd(String cmd) throws Exception { } public void createSources() throws Exception { - - startSessionState(); + if(!isSessionStateStarted) { + startSessionState(); + } conf.setBoolean("hive.test.init.phase", true); - // Create a bunch of tables with columns key and value - LinkedList cols = new LinkedList(); - cols.add("key"); - cols.add("value"); - - LinkedList part_cols = new LinkedList(); - part_cols.add("ds"); - part_cols.add("hr"); - db.createTable("srcpart", cols, part_cols, TextInputFormat.class, - IgnoreKeyTextOutputFormat.class); - - Path fpath; - HashMap part_spec = new HashMap(); - for (String ds : new String[] {"2008-04-08", "2008-04-09"}) { - for (String hr : new String[] {"11", "12"}) { - part_spec.clear(); - part_spec.put("ds", ds); - part_spec.put("hr", hr); - // System.out.println("Loading partition with spec: " + part_spec); - // db.createPartition(srcpart, part_spec); - fpath = new Path(testFiles, "kv1.txt"); - // db.loadPartition(fpath, srcpart.getName(), part_spec, true); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() - + "' OVERWRITE INTO TABLE srcpart PARTITION (ds='" + ds + "',hr='" - + hr + "')"); - } + final String initScript = "../../data/scripts/q_test_init.sql"; + String initCommands = readEntireFileIntoString(new File(initScript)); + LOG.info("Initial setup:\n" + initCommands); + if(cliDriver == null) { + cliDriver = new CliDriver(); } - ArrayList bucketCols = new ArrayList(); - bucketCols.add("key"); - runCreateTableCmd("CREATE TABLE srcbucket(key int, value string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE"); - // db.createTable("srcbucket", cols, null, TextInputFormat.class, - // IgnoreKeyTextOutputFormat.class, 2, bucketCols); - for (String fname : new String[] {"srcbucket0.txt", "srcbucket1.txt"}) { - fpath = new Path(testFiles, fname); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() - + "' INTO TABLE srcbucket"); - } - - runCreateTableCmd("CREATE TABLE srcbucket2(key int, value string) " - + "CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE"); - // db.createTable("srcbucket", cols, null, TextInputFormat.class, - // IgnoreKeyTextOutputFormat.class, 2, bucketCols); - for (String fname : new String[] {"srcbucket20.txt", "srcbucket21.txt", - "srcbucket22.txt", "srcbucket23.txt"}) { - fpath = new Path(testFiles, fname); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() - + "' INTO TABLE srcbucket2"); - } - - for (String tname : new String[] {"src", "src1"}) { - db.createTable(tname, cols, null, TextInputFormat.class, - IgnoreKeyTextOutputFormat.class); - } - db.createTable("src_sequencefile", cols, null, - SequenceFileInputFormat.class, SequenceFileOutputFormat.class); - - Table srcThrift = - new Table(SessionState.get().getCurrentDatabase(), "src_thrift"); - srcThrift.setInputFormatClass(SequenceFileInputFormat.class.getName()); - srcThrift.setOutputFormatClass(SequenceFileOutputFormat.class.getName()); - srcThrift.setSerializationLib(ThriftDeserializer.class.getName()); - srcThrift.setSerdeParam(serdeConstants.SERIALIZATION_CLASS, Complex.class - .getName()); - srcThrift.setSerdeParam(serdeConstants.SERIALIZATION_FORMAT, - TBinaryProtocol.class.getName()); - db.createTable(srcThrift); - - LinkedList json_cols = new LinkedList(); - json_cols.add("json"); - db.createTable("src_json", json_cols, null, TextInputFormat.class, - IgnoreKeyTextOutputFormat.class); - - // load the input data into the src table - fpath = new Path(testFiles, "kv1.txt"); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' INTO TABLE src"); - - // load the input data into the src table - fpath = new Path(testFiles, "kv3.txt"); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() + "' INTO TABLE src1"); - - // load the input data into the src_sequencefile table - fpath = new Path(testFiles, "kv1.seq"); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() - + "' INTO TABLE src_sequencefile"); - - // load the input data into the src_thrift table - fpath = new Path(testFiles, "complex.seq"); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() - + "' INTO TABLE src_thrift"); - - // load the json data into the src_json table - fpath = new Path(testFiles, "json.txt"); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() - + "' INTO TABLE src_json"); - - FileSystem localFs = FileSystem.getLocal(conf); - // create and load data into orc table - fpath = new Path(testFiles, AllVectorTypesRecord.TABLE_NAME); - - runCreateTableCmd(AllVectorTypesRecord.TABLE_CREATE_COMMAND); - runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toUri().getPath() - + "' INTO TABLE "+AllVectorTypesRecord.TABLE_NAME); - - runCmd("DROP FUNCTION IF EXISTS qtest_get_java_boolean "); - runCmd("CREATE FUNCTION qtest_get_java_boolean " - + " AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaBoolean'"); + cliDriver.processLine(initCommands); conf.setBoolean("hive.test.init.phase", false); - } public void init() throws Exception { @@ -786,33 +689,6 @@ public void init() throws Exception { public void init(String tname) throws Exception { cleanUp(); createSources(); - - LinkedList cols = new LinkedList(); - cols.add("key"); - cols.add("value"); - - LinkedList part_cols = new LinkedList(); - part_cols.add("ds"); - part_cols.add("hr"); - - db.createTable("dest1", cols, null, TextInputFormat.class, - IgnoreKeyTextOutputFormat.class); - db.createTable("dest2", cols, null, TextInputFormat.class, - IgnoreKeyTextOutputFormat.class); - - db.createTable("dest3", cols, part_cols, TextInputFormat.class, - IgnoreKeyTextOutputFormat.class); - Table dest3 = db.getTable("dest3"); - - HashMap part_spec = new HashMap(); - part_spec.put("ds", "2008-04-08"); - part_spec.put("hr", "12"); - db.createPartition(dest3, part_spec); - - db.createTable("dest4", cols, null, TextInputFormat.class, - IgnoreKeyTextOutputFormat.class); - db.createTable("dest4_sequencefile", cols, null, - SequenceFileInputFormat.class, SequenceFileOutputFormat.class); } public void cliInit(String tname) throws Exception { @@ -865,24 +741,40 @@ public String cliInit(String tname, boolean recreate) throws Exception { } SessionState.start(ss); - cliDriver = new CliDriver(); + if(cliDriver == null) { + cliDriver = new CliDriver(); + } if (tname.equals("init_file.q")) { ss.initFiles.add("../../data/scripts/test_init_file.sql"); } cliDriver.processInitFiles(ss); + return outf.getAbsolutePath(); } private CliSessionState startSessionState() - throws FileNotFoundException, UnsupportedEncodingException { + throws IOException { HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER, "org.apache.hadoop.hive.ql.security.DummyAuthenticator"); CliSessionState ss = new CliSessionState(conf); assert ss != null; + ss.in = System.in; + ss.out = System.out; + ss.err = System.out; + SessionState oldSs = SessionState.get(); + if (oldSs != null && clusterType == MiniClusterType.tez) { + oldSs.close(); + } + if (oldSs != null && oldSs.out != null && oldSs.out != System.out) { + oldSs.out.close(); + } SessionState.start(ss); + + isSessionStateStarted = true; + return ss; }