commit 8db94b9bd8e5a07871c123eaf120cceadaec737e Author: Abdullah Yousufi Date: Tue Aug 9 15:21:27 2016 -0700 HIVE-14373: Add integration tests for hive on S3 diff --git a/itests/hive-blobstore/pom.xml b/itests/hive-blobstore/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..0d5917196c11a2fa1f266770e74cf2e7e4552c0f --- /dev/null +++ b/itests/hive-blobstore/pom.xml @@ -0,0 +1,257 @@ + + + + 4.0.0 + + + org.apache.hive + hive-it + 2.2.0-SNAPSHOT + ../pom.xml + + + hive-blobstore + jar + Hive Integration - Blobstore Tests + + + ../.. + + + + false + false + ${hadoop.version} + -mkdir -p + + + + + tests-off + + + src/test/resources/auth-keys.xml + + + + true + + + + tests-on + + + src/test/resources/auth-keys.xml + + + + false + + + + + + + + org.apache.hive + hive-ant + ${project.version} + test + + + org.apache.hive + hive-common + ${project.version} + test + + + org.apache.hive + hive-contrib + ${project.version} + test + + + org.apache.hive + hive-metastore + ${project.version} + test + + + org.apache.hive + hive-metastore + ${project.version} + tests + test + + + org.apache.hive + hive-it-unit + ${project.version} + tests + test + + + org.apache.hive + hive-serde + ${project.version} + test + + + org.apache.hive + hive-exec + ${project.version} + test + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + test + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + tests + test + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + ${hadoop.version} + tests + test + + + org.apache.hadoop + hadoop-mapreduce-client-hs + ${hadoop.version} + test + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + test + + + junit + junit + ${junit.version} + test + + + org.apache.hadoop + hadoop-aws + 2.6.0 + compile + + + com.fasterxml.jackson.core + jackson-annotations + 2.2.3 + + + com.fasterxml.jackson.core + jackson-core + 2.2.3 + + + com.fasterxml.jackson.core + jackson-databind + 2.2.3 + + + + + + + org.codehaus.mojo + properties-maven-plugin + 1.0-alpha-2 + + + initialize + + read-project-properties + + + + ${basedir}/../src/test/resources/testconfiguration.properties + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + generate-tests-sources + generate-test-sources + + + + + + + + + + + + + + run + + + + + + org.codehaus.mojo + build-helper-maven-plugin + ${maven.build-helper.plugin.version} + + + add-test-sources + generate-test-sources + + add-test-source + + + + target/generated-test-sources/java + + + + + + + + + diff --git a/itests/hive-blobstore/queries/test.q b/itests/hive-blobstore/queries/test.q new file mode 100644 index 0000000000000000000000000000000000000000..7ddc720fbf126bb66e665d422f9ca4bd6622044e --- /dev/null +++ b/itests/hive-blobstore/queries/test.q @@ -0,0 +1,4 @@ +create external table qtest (value int) location '${hiveconf:fs.s3a.bucket.name}/user/hive/warehouse/qtest/'; +insert into qtest values (1), (10), (100), (1000); +select * from qtest; +drop table qtest purge; diff --git a/itests/hive-blobstore/results/test.q.out b/itests/hive-blobstore/results/test.q.out new file mode 100644 index 0000000000000000000000000000000000000000..54063d9d5e665b1a8245a71ec382c988d18b9caa --- /dev/null +++ b/itests/hive-blobstore/results/test.q.out @@ -0,0 +1,39 @@ +PREHOOK: query: create external table qtest (value int) location '#### A masked pattern was here ####' +PREHOOK: type: CREATETABLE +PREHOOK: Input: #### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@qtest +POSTHOOK: query: create external table qtest (value int) location '#### A masked pattern was here ####' +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: #### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@qtest +PREHOOK: query: insert into qtest values (1), (10), (100), (1000) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@qtest +POSTHOOK: query: insert into qtest values (1), (10), (100), (1000) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@qtest +POSTHOOK: Lineage: qtest.value EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: select * from qtest +PREHOOK: type: QUERY +PREHOOK: Input: default@qtest +#### A masked pattern was here #### +POSTHOOK: query: select * from qtest +POSTHOOK: type: QUERY +POSTHOOK: Input: default@qtest +#### A masked pattern was here #### +1 +10 +100 +1000 +PREHOOK: query: drop table qtest purge +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@qtest +PREHOOK: Output: default@qtest +POSTHOOK: query: drop table qtest purge +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@qtest +POSTHOOK: Output: default@qtest diff --git a/itests/hive-blobstore/src/test/resources/auth-keys.xml.template b/itests/hive-blobstore/src/test/resources/auth-keys.xml.template new file mode 100644 index 0000000000000000000000000000000000000000..50d1e8296d010e68a3fc9ba8b43a1856113d9715 --- /dev/null +++ b/itests/hive-blobstore/src/test/resources/auth-keys.xml.template @@ -0,0 +1,16 @@ + + + + + fs.s3a.access.key + + + + fs.s3a.secret.key + + + + fs.s3a.bucket.name + + + diff --git a/itests/hive-blobstore/src/test/resources/core-site.xml b/itests/hive-blobstore/src/test/resources/core-site.xml new file mode 100644 index 0000000000000000000000000000000000000000..57469527418d363322b2d22bd00ed68bca28e2d6 --- /dev/null +++ b/itests/hive-blobstore/src/test/resources/core-site.xml @@ -0,0 +1,49 @@ + + + + + + + + + + hadoop.tmp.dir + target/build/test + A base for other temporary directories. + true + + + + + hadoop.security.authentication + simple + + + + + + + + diff --git a/itests/pom.xml b/itests/pom.xml index 426ba04bd8131e24dc7e33056cd32087fcf3662c..14dfad335e8f057fe3df92fcac71b895845dba67 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -36,6 +36,7 @@ custom-udfs hcatalog-unit hive-unit + hive-blobstore util test-serde qtest diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index b43c4a73b083542ea799d2a0305665a4ee0a09d3..49f5caa7e459ddfc29476b96aff8593cd677dfae 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -1479,7 +1479,8 @@ private void maskPatterns(Pattern[] patterns, String fname) throws Exception { "fk_-?[0-9]*_[0-9]*_[0-9]*", ".*at com\\.sun\\.proxy.*", ".*at com\\.jolbox.*", - "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*" + "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*", + "s3a:\\/\\/[\\w\\.\\/-]*" }); private final Pattern[] partialReservedPlanMask = toPattern(new String[] { @@ -2158,7 +2159,7 @@ public int compare(String str1, String str2) { int i = 0; while (!statements.isEmpty()) { // PreparedStatement extend Statement - Statement st = (Statement)statements.remove(i); + Statement st = statements.remove(i); try { if (st != null) { st.close();