diff --git itests/hive-blobstore/pom.xml itests/hive-blobstore/pom.xml new file mode 100644 index 0000000..bb73042 --- /dev/null +++ itests/hive-blobstore/pom.xml @@ -0,0 +1,377 @@ + + + + 4.0.0 + + + org.apache.hive + hive-it + 2.2.0-SNAPSHOT + ../pom.xml + + + hive-blobstore + jar + Hive Integration - Blobstore Tests + + + ../.. + + + false + + false + ${hadoop.version} + -mkdir -p + + + + + tests-off + + + src/test/resources/blobstore-conf.xml + + + + true + + + + tests-on + + + src/test/resources/blobstore-conf.xml + + + + false + + + + + + + + org.apache.hive + hive-ant + ${project.version} + test + + + org.apache.hive + hive-common + ${project.version} + test + + + org.apache.hive + hive-contrib + ${project.version} + test + + + org.apache.hive + hive-metastore + ${project.version} + test + + + org.apache.hive + hive-metastore + ${project.version} + tests + test + + + org.apache.hive + hive-it-unit + ${project.version} + tests + test + + + org.apache.hive + hive-serde + ${project.version} + test + + + org.apache.hive + hive-exec + ${project.version} + test + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + test + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + tests + test + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + ${hadoop.version} + tests + test + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + + + org.apache.hadoop + hadoop-mapreduce-client-hs + ${hadoop.version} + test + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + test + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + + + org.apache.tez + tez-tests + ${tez.version} + test-jar + + + org.apache.tez + tez-api + ${tez.version} + test + + + org.apache.tez + tez-runtime-library + ${tez.version} + test + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + + + org.apache.tez + tez-mapreduce + ${tez.version} + test + + + org.apache.tez + tez-dag + ${tez.version} + test + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + + + junit + junit + ${junit.version} + test + + + org.apache.hadoop + hadoop-aws + ${hadoop.version} + compile + + + org.slf4j + slf4j-log4j12 + + + commmons-logging + commons-logging + + + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.new.version} + + + com.fasterxml.jackson.core + jackson-core + ${jackson.new.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.new.version} + + + + + + + org.codehaus.mojo + properties-maven-plugin + 1.0-alpha-2 + + + initialize + + read-project-properties + + + + ${basedir}/../src/test/resources/testconfiguration.properties + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + generate-tests-sources + generate-test-sources + + + + + + + + + + + + + + + + + run + + + + + + org.codehaus.mojo + build-helper-maven-plugin + ${maven.build-helper.plugin.version} + + + add-test-sources + generate-test-sources + + add-test-source + + + + target/generated-test-sources/java + + + + + + + + + diff --git itests/hive-blobstore/src/test/queries/clientnegative/select_table_not_exists.q itests/hive-blobstore/src/test/queries/clientnegative/select_table_not_exists.q new file mode 100644 index 0000000..6084cc6 --- /dev/null +++ itests/hive-blobstore/src/test/queries/clientnegative/select_table_not_exists.q @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS qtest; +select * from qtest; diff --git itests/hive-blobstore/src/test/queries/clientpositive/insert_values_into_table.q itests/hive-blobstore/src/test/queries/clientpositive/insert_values_into_table.q new file mode 100644 index 0000000..7dce3c9 --- /dev/null +++ itests/hive-blobstore/src/test/queries/clientpositive/insert_values_into_table.q @@ -0,0 +1,4 @@ +CREATE TABLE qtest (value int) LOCATION '${hiveconf:fs.blobstore.location}/qtest/'; +INSERT INTO qtest VALUES (1), (10), (100), (1000); +SELECT * FROM qtest; +DROP TABLE qtest; diff --git itests/hive-blobstore/src/test/resources/blobstore-conf.xml.template itests/hive-blobstore/src/test/resources/blobstore-conf.xml.template new file mode 100644 index 0000000..ac7b940 --- /dev/null +++ itests/hive-blobstore/src/test/resources/blobstore-conf.xml.template @@ -0,0 +1,22 @@ + + + + + + fs.s3a.access.key + + + + fs.s3a.secret.key + + + + fs.blobstore.location + s3a://example-bucket/user/hive/warehouse + + diff --git itests/hive-blobstore/src/test/resources/core-site.xml itests/hive-blobstore/src/test/resources/core-site.xml new file mode 100644 index 0000000..8c41135 --- /dev/null +++ itests/hive-blobstore/src/test/resources/core-site.xml @@ -0,0 +1,48 @@ + + + + + + + + + + hadoop.tmp.dir + target/build/test + A base for other temporary directories. + true + + + + + hadoop.security.authentication + simple + + + + + + + + diff --git itests/hive-blobstore/src/test/results/clientnegative/select_table_not_exists.q.out itests/hive-blobstore/src/test/results/clientnegative/select_table_not_exists.q.out new file mode 100644 index 0000000..6800c3a --- /dev/null +++ itests/hive-blobstore/src/test/results/clientnegative/select_table_not_exists.q.out @@ -0,0 +1,5 @@ +PREHOOK: query: DROP TABLE IF EXISTS qtest +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS qtest +POSTHOOK: type: DROPTABLE +FAILED: SemanticException [Error 10001]: Line 2:14 Table not found 'qtest' diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_values_into_table.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_values_into_table.q.out new file mode 100644 index 0000000..12f03e9 --- /dev/null +++ itests/hive-blobstore/src/test/results/clientpositive/insert_values_into_table.q.out @@ -0,0 +1,39 @@ +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: #### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@qtest +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: #### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@qtest +PREHOOK: query: INSERT INTO qtest VALUES (1), (10), (100), (1000) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@qtest +POSTHOOK: query: INSERT INTO qtest VALUES (1), (10), (100), (1000) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@qtest +POSTHOOK: Lineage: qtest.value EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: SELECT * FROM qtest +PREHOOK: type: QUERY +PREHOOK: Input: default@qtest +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM qtest +POSTHOOK: type: QUERY +POSTHOOK: Input: default@qtest +#### A masked pattern was here #### +1 +10 +100 +1000 +PREHOOK: query: DROP TABLE qtest +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@qtest +PREHOOK: Output: default@qtest +POSTHOOK: query: DROP TABLE qtest +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@qtest +POSTHOOK: Output: default@qtest diff --git itests/hive-blobstore/src/test/templates/TestCliDriver.vm itests/hive-blobstore/src/test/templates/TestCliDriver.vm new file mode 100644 index 0000000..53ac1d3 --- /dev/null +++ itests/hive-blobstore/src/test/templates/TestCliDriver.vm @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.cli; + +import org.apache.hadoop.hive.ql.QTestUtil; +import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class $className { + + private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); + private static QTestUtil qt; + + static { + + MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode"); + String hiveConfDir = "$hiveConfDir"; + String initScript = "$initScript"; + String cleanupScript = "$cleanupScript"; + boolean useHBaseMetastore = Boolean.valueOf("$useHBaseMetastore"); + try { + String hadoopVer = "$hadoopVersion"; + if (!hiveConfDir.isEmpty()) { + hiveConfDir = HIVE_ROOT + hiveConfDir; + } + qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, + hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true); + + // do a one time initialization + qt.cleanUp(); + qt.createSources(); + + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception in static initialization: "+e.getMessage()); + } + } + + @Before + public void setUp() { + try { + qt.clearTestSideEffects(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception in setup"); + } + } + + @After + public void tearDown() { + try { + qt.clearPostTestEffects(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception in tearDown"); + } + } + + @AfterClass + public static void shutdown() throws Exception { + try { + qt.shutdown(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception in shutdown"); + } + } + + static String debugHint = "\nSee ./itests/hive-blobstore/target/tmp/log/hive.log " + + "or check ./itests/hive-blobstore/target/surefire-reports/ for specific test cases logs."; + +#foreach ($qf in $qfiles) + #set ($fname = $qf.getName()) + #set ($eidx = $fname.indexOf('.')) + #set ($tname = $fname.substring(0, $eidx)) + #set ($fpath = $qfilesMap.get($fname)) + @Test + public void testCliDriver_$tname() throws Exception { + runTest("$tname", "$fname", (HIVE_ROOT + "$fpath")); + } + +#end + + private void runTest(String tname, String fname, String fpath) throws Exception { + long startTime = System.currentTimeMillis(); + try { + System.err.println("Begin query: " + fname); + + qt.addFile(fpath); + + if (qt.shouldBeSkipped(fname)) { + System.err.println("Test " + fname + " skipped"); + return; + } + + qt.cliInit(fname, false); + int ecode = qt.executeClient(fname); + if (ecode != 0) { + qt.failed(ecode, fname, debugHint); + } + ecode = qt.checkCliDriverResults(fname); + if (ecode != 0) { + qt.failedDiff(ecode, fname, debugHint); + } + } + catch (Throwable e) { + qt.failed(e, fname, debugHint); + } + + long elapsedTime = System.currentTimeMillis() - startTime; + System.err.println("Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s"); + assertTrue("Test passed", true); + } +} diff --git itests/hive-blobstore/src/test/templates/TestNegativeCliDriver.vm itests/hive-blobstore/src/test/templates/TestNegativeCliDriver.vm new file mode 100644 index 0000000..d06bded --- /dev/null +++ itests/hive-blobstore/src/test/templates/TestNegativeCliDriver.vm @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.cli; + +import org.apache.hadoop.hive.ql.QTestUtil; +import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class $className { + + private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); + private static QTestUtil qt; + + static { + MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode"); + String hiveConfDir = "$hiveConfDir"; + String initScript = "$initScript"; + String cleanupScript = "$cleanupScript"; + + try { + String hadoopVer = "$hadoopVersion"; + if (!hiveConfDir.isEmpty()) { + hiveConfDir = HIVE_ROOT + hiveConfDir; + } + qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, + hiveConfDir, hadoopVer, initScript, cleanupScript, false, false); + // do a one time initialization + qt.cleanUp(); + qt.createSources(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception in static initialization"); + } + } + + @Before + public void setUp() { + try { + qt.clearTestSideEffects(); + } catch (Throwable e) { + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception in setup"); + } + } + + @After + public void tearDown() { + try { + qt.clearPostTestEffects(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception in tearDown"); + } + } + + @AfterClass + public static void shutdown() throws Exception { + try { + qt.shutdown(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception in shutdown"); + } + } + + /** + * Dummy last test. This is only meant to shutdown qt + */ + public void testNegativeCliDriver_shutdown() { + System.err.println ("Cleaning up " + "$className"); + } + + static String debugHint = "\nSee ./itests/hive-blobstore/target/tmp/log/hive.log " + + "or check ./itests/hive-blobstore/target/surefire-reports/ for specific test cases logs."; + +#foreach ($qf in $qfiles) + #set ($fname = $qf.getName()) + #set ($eidx = $fname.indexOf('.')) + #set ($tname = $fname.substring(0, $eidx)) + #set ($fpath = $qfilesMap.get($fname)) + @Test + public void testNegativeCliDriver_$tname() throws Exception { + runTest("$tname", "$fname", (HIVE_ROOT + "$fpath")); + } + +#end + + private void runTest(String tname, String fname, String fpath) throws Exception { + long startTime = System.currentTimeMillis(); + try { + System.err.println("Begin query: " + fname); + + qt.addFile(fpath); + + if (qt.shouldBeSkipped(fname)) { + System.err.println("Test " + fname + " skipped"); + return; + } + + qt.cliInit(fname, false); + int ecode = qt.executeClient(fname); + if (ecode == 0) { + qt.failed(fname, debugHint); + } + + ecode = qt.checkCliDriverResults(fname); + if (ecode != 0) { + qt.failedDiff(ecode, fname, debugHint); + } + } + catch (Throwable e) { + qt.failed(e, fname, debugHint); + } + + long elapsedTime = System.currentTimeMillis() - startTime; + System.err.println("Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s"); + assertTrue("Test passed", true); + } +} diff --git itests/pom.xml itests/pom.xml index 426ba04..14dfad3 100644 --- itests/pom.xml +++ itests/pom.xml @@ -36,6 +36,7 @@ custom-udfs hcatalog-unit hive-unit + hive-blobstore util test-serde qtest diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index b43c4a7..704edbf 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -1479,7 +1479,8 @@ private void maskPatterns(Pattern[] patterns, String fname) throws Exception { "fk_-?[0-9]*_[0-9]*_[0-9]*", ".*at com\\.sun\\.proxy.*", ".*at com\\.jolbox.*", - "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*" + "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*", + "(s3.?|swift|wasb.?):\\/\\/[\\w\\.\\/-]*" }); private final Pattern[] partialReservedPlanMask = toPattern(new String[] { @@ -2158,7 +2159,7 @@ public int compare(String str1, String str2) { int i = 0; while (!statements.isEmpty()) { // PreparedStatement extend Statement - Statement st = (Statement)statements.remove(i); + Statement st = statements.remove(i); try { if (st != null) { st.close();