diff --git itests/hive-blobstore/pom.xml itests/hive-blobstore/pom.xml
new file mode 100644
index 0000000..bb73042
--- /dev/null
+++ itests/hive-blobstore/pom.xml
@@ -0,0 +1,377 @@
+
+
+
+ 4.0.0
+
+
+ org.apache.hive
+ hive-it
+ 2.2.0-SNAPSHOT
+ ../pom.xml
+
+
+ hive-blobstore
+ jar
+ Hive Integration - Blobstore Tests
+
+
+ ../..
+
+
+ false
+
+ false
+ ${hadoop.version}
+ -mkdir -p
+
+
+
+
+ tests-off
+
+
+ src/test/resources/blobstore-conf.xml
+
+
+
+ true
+
+
+
+ tests-on
+
+
+ src/test/resources/blobstore-conf.xml
+
+
+
+ false
+
+
+
+
+
+
+
+ org.apache.hive
+ hive-ant
+ ${project.version}
+ test
+
+
+ org.apache.hive
+ hive-common
+ ${project.version}
+ test
+
+
+ org.apache.hive
+ hive-contrib
+ ${project.version}
+ test
+
+
+ org.apache.hive
+ hive-metastore
+ ${project.version}
+ test
+
+
+ org.apache.hive
+ hive-metastore
+ ${project.version}
+ tests
+ test
+
+
+ org.apache.hive
+ hive-it-unit
+ ${project.version}
+ tests
+ test
+
+
+ org.apache.hive
+ hive-serde
+ ${project.version}
+ test
+
+
+ org.apache.hive
+ hive-exec
+ ${project.version}
+ test
+
+
+
+ org.apache.hadoop
+ hadoop-common
+ ${hadoop.version}
+ test
+
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ commmons-logging
+ commons-logging
+
+
+
+
+ org.apache.hadoop
+ hadoop-common
+ ${hadoop.version}
+ tests
+ test
+
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ commmons-logging
+ commons-logging
+
+
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-jobclient
+ ${hadoop.version}
+ tests
+ test
+
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ commmons-logging
+ commons-logging
+
+
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-hs
+ ${hadoop.version}
+ test
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-core
+ ${hadoop.version}
+ test
+
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ commmons-logging
+ commons-logging
+
+
+
+
+ org.apache.tez
+ tez-tests
+ ${tez.version}
+ test-jar
+
+
+ org.apache.tez
+ tez-api
+ ${tez.version}
+ test
+
+
+ org.apache.tez
+ tez-runtime-library
+ ${tez.version}
+ test
+
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ commmons-logging
+ commons-logging
+
+
+
+
+ org.apache.tez
+ tez-mapreduce
+ ${tez.version}
+ test
+
+
+ org.apache.tez
+ tez-dag
+ ${tez.version}
+ test
+
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ commmons-logging
+ commons-logging
+
+
+
+
+ junit
+ junit
+ ${junit.version}
+ test
+
+
+ org.apache.hadoop
+ hadoop-aws
+ ${hadoop.version}
+ compile
+
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ commmons-logging
+ commons-logging
+
+
+
+
+ com.fasterxml.jackson.core
+ jackson-annotations
+ ${jackson.new.version}
+
+
+ com.fasterxml.jackson.core
+ jackson-core
+ ${jackson.new.version}
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+ ${jackson.new.version}
+
+
+
+
+
+
+ org.codehaus.mojo
+ properties-maven-plugin
+ 1.0-alpha-2
+
+
+ initialize
+
+ read-project-properties
+
+
+
+ ${basedir}/../src/test/resources/testconfiguration.properties
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-antrun-plugin
+
+
+ generate-tests-sources
+ generate-test-sources
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ run
+
+
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+ ${maven.build-helper.plugin.version}
+
+
+ add-test-sources
+ generate-test-sources
+
+ add-test-source
+
+
+
+ target/generated-test-sources/java
+
+
+
+
+
+
+
+
+
diff --git itests/hive-blobstore/src/test/queries/clientnegative/select_table_not_exists.q itests/hive-blobstore/src/test/queries/clientnegative/select_table_not_exists.q
new file mode 100644
index 0000000..6084cc6
--- /dev/null
+++ itests/hive-blobstore/src/test/queries/clientnegative/select_table_not_exists.q
@@ -0,0 +1,2 @@
+DROP TABLE IF EXISTS qtest;
+select * from qtest;
diff --git itests/hive-blobstore/src/test/queries/clientpositive/insert_values_into_table.q itests/hive-blobstore/src/test/queries/clientpositive/insert_values_into_table.q
new file mode 100644
index 0000000..7dce3c9
--- /dev/null
+++ itests/hive-blobstore/src/test/queries/clientpositive/insert_values_into_table.q
@@ -0,0 +1,4 @@
+CREATE TABLE qtest (value int) LOCATION '${hiveconf:fs.blobstore.location}/qtest/';
+INSERT INTO qtest VALUES (1), (10), (100), (1000);
+SELECT * FROM qtest;
+DROP TABLE qtest;
diff --git itests/hive-blobstore/src/test/resources/blobstore-conf.xml.template itests/hive-blobstore/src/test/resources/blobstore-conf.xml.template
new file mode 100644
index 0000000..ac7b940
--- /dev/null
+++ itests/hive-blobstore/src/test/resources/blobstore-conf.xml.template
@@ -0,0 +1,22 @@
+
+
+
+
+
+ fs.s3a.access.key
+
+
+
+ fs.s3a.secret.key
+
+
+
+ fs.blobstore.location
+ s3a://example-bucket/user/hive/warehouse
+
+
diff --git itests/hive-blobstore/src/test/resources/core-site.xml itests/hive-blobstore/src/test/resources/core-site.xml
new file mode 100644
index 0000000..8c41135
--- /dev/null
+++ itests/hive-blobstore/src/test/resources/core-site.xml
@@ -0,0 +1,48 @@
+
+
+
+
+
+
+
+
+
+ hadoop.tmp.dir
+ target/build/test
+ A base for other temporary directories.
+ true
+
+
+
+
+ hadoop.security.authentication
+ simple
+
+
+
+
+
+
+
+
diff --git itests/hive-blobstore/src/test/results/clientnegative/select_table_not_exists.q.out itests/hive-blobstore/src/test/results/clientnegative/select_table_not_exists.q.out
new file mode 100644
index 0000000..6800c3a
--- /dev/null
+++ itests/hive-blobstore/src/test/results/clientnegative/select_table_not_exists.q.out
@@ -0,0 +1,5 @@
+PREHOOK: query: DROP TABLE IF EXISTS qtest
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS qtest
+POSTHOOK: type: DROPTABLE
+FAILED: SemanticException [Error 10001]: Line 2:14 Table not found 'qtest'
diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_values_into_table.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_values_into_table.q.out
new file mode 100644
index 0000000..12f03e9
--- /dev/null
+++ itests/hive-blobstore/src/test/results/clientpositive/insert_values_into_table.q.out
@@ -0,0 +1,39 @@
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: #### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@qtest
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: #### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@qtest
+PREHOOK: query: INSERT INTO qtest VALUES (1), (10), (100), (1000)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@qtest
+POSTHOOK: query: INSERT INTO qtest VALUES (1), (10), (100), (1000)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@qtest
+POSTHOOK: Lineage: qtest.value EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: SELECT * FROM qtest
+PREHOOK: type: QUERY
+PREHOOK: Input: default@qtest
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM qtest
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@qtest
+#### A masked pattern was here ####
+1
+10
+100
+1000
+PREHOOK: query: DROP TABLE qtest
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@qtest
+PREHOOK: Output: default@qtest
+POSTHOOK: query: DROP TABLE qtest
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@qtest
+POSTHOOK: Output: default@qtest
diff --git itests/hive-blobstore/src/test/templates/TestCliDriver.vm itests/hive-blobstore/src/test/templates/TestCliDriver.vm
new file mode 100644
index 0000000..53ac1d3
--- /dev/null
+++ itests/hive-blobstore/src/test/templates/TestCliDriver.vm
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.cli;
+
+import org.apache.hadoop.hive.ql.QTestUtil;
+import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class $className {
+
+ private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root"));
+ private static QTestUtil qt;
+
+ static {
+
+ MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
+ String hiveConfDir = "$hiveConfDir";
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";
+ boolean useHBaseMetastore = Boolean.valueOf("$useHBaseMetastore");
+ try {
+ String hadoopVer = "$hadoopVersion";
+ if (!hiveConfDir.isEmpty()) {
+ hiveConfDir = HIVE_ROOT + hiveConfDir;
+ }
+ qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
+ hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true);
+
+ // do a one time initialization
+ qt.cleanUp();
+ qt.createSources();
+
+ } catch (Exception e) {
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.err.flush();
+ fail("Unexpected exception in static initialization: "+e.getMessage());
+ }
+ }
+
+ @Before
+ public void setUp() {
+ try {
+ qt.clearTestSideEffects();
+ } catch (Exception e) {
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.err.flush();
+ fail("Unexpected exception in setup");
+ }
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ qt.clearPostTestEffects();
+ } catch (Exception e) {
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.err.flush();
+ fail("Unexpected exception in tearDown");
+ }
+ }
+
+ @AfterClass
+ public static void shutdown() throws Exception {
+ try {
+ qt.shutdown();
+ } catch (Exception e) {
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.err.flush();
+ fail("Unexpected exception in shutdown");
+ }
+ }
+
+ static String debugHint = "\nSee ./itests/hive-blobstore/target/tmp/log/hive.log "
+ + "or check ./itests/hive-blobstore/target/surefire-reports/ for specific test cases logs.";
+
+#foreach ($qf in $qfiles)
+ #set ($fname = $qf.getName())
+ #set ($eidx = $fname.indexOf('.'))
+ #set ($tname = $fname.substring(0, $eidx))
+ #set ($fpath = $qfilesMap.get($fname))
+ @Test
+ public void testCliDriver_$tname() throws Exception {
+ runTest("$tname", "$fname", (HIVE_ROOT + "$fpath"));
+ }
+
+#end
+
+ private void runTest(String tname, String fname, String fpath) throws Exception {
+ long startTime = System.currentTimeMillis();
+ try {
+ System.err.println("Begin query: " + fname);
+
+ qt.addFile(fpath);
+
+ if (qt.shouldBeSkipped(fname)) {
+ System.err.println("Test " + fname + " skipped");
+ return;
+ }
+
+ qt.cliInit(fname, false);
+ int ecode = qt.executeClient(fname);
+ if (ecode != 0) {
+ qt.failed(ecode, fname, debugHint);
+ }
+ ecode = qt.checkCliDriverResults(fname);
+ if (ecode != 0) {
+ qt.failedDiff(ecode, fname, debugHint);
+ }
+ }
+ catch (Throwable e) {
+ qt.failed(e, fname, debugHint);
+ }
+
+ long elapsedTime = System.currentTimeMillis() - startTime;
+ System.err.println("Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s");
+ assertTrue("Test passed", true);
+ }
+}
diff --git itests/hive-blobstore/src/test/templates/TestNegativeCliDriver.vm itests/hive-blobstore/src/test/templates/TestNegativeCliDriver.vm
new file mode 100644
index 0000000..d06bded
--- /dev/null
+++ itests/hive-blobstore/src/test/templates/TestNegativeCliDriver.vm
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.cli;
+
+import org.apache.hadoop.hive.ql.QTestUtil;
+import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class $className {
+
+ private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root"));
+ private static QTestUtil qt;
+
+ static {
+ MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
+ String hiveConfDir = "$hiveConfDir";
+ String initScript = "$initScript";
+ String cleanupScript = "$cleanupScript";
+
+ try {
+ String hadoopVer = "$hadoopVersion";
+ if (!hiveConfDir.isEmpty()) {
+ hiveConfDir = HIVE_ROOT + hiveConfDir;
+ }
+ qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
+ hiveConfDir, hadoopVer, initScript, cleanupScript, false, false);
+ // do a one time initialization
+ qt.cleanUp();
+ qt.createSources();
+ } catch (Exception e) {
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.err.flush();
+ fail("Unexpected exception in static initialization");
+ }
+ }
+
+ @Before
+ public void setUp() {
+ try {
+ qt.clearTestSideEffects();
+ } catch (Throwable e) {
+ e.printStackTrace();
+ System.err.flush();
+ fail("Unexpected exception in setup");
+ }
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ qt.clearPostTestEffects();
+ } catch (Exception e) {
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.err.flush();
+ fail("Unexpected exception in tearDown");
+ }
+ }
+
+ @AfterClass
+ public static void shutdown() throws Exception {
+ try {
+ qt.shutdown();
+ } catch (Exception e) {
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.err.flush();
+ fail("Unexpected exception in shutdown");
+ }
+ }
+
+ /**
+ * Dummy last test. This is only meant to shutdown qt
+ */
+ public void testNegativeCliDriver_shutdown() {
+ System.err.println ("Cleaning up " + "$className");
+ }
+
+ static String debugHint = "\nSee ./itests/hive-blobstore/target/tmp/log/hive.log "
+ + "or check ./itests/hive-blobstore/target/surefire-reports/ for specific test cases logs.";
+
+#foreach ($qf in $qfiles)
+ #set ($fname = $qf.getName())
+ #set ($eidx = $fname.indexOf('.'))
+ #set ($tname = $fname.substring(0, $eidx))
+ #set ($fpath = $qfilesMap.get($fname))
+ @Test
+ public void testNegativeCliDriver_$tname() throws Exception {
+ runTest("$tname", "$fname", (HIVE_ROOT + "$fpath"));
+ }
+
+#end
+
+ private void runTest(String tname, String fname, String fpath) throws Exception {
+ long startTime = System.currentTimeMillis();
+ try {
+ System.err.println("Begin query: " + fname);
+
+ qt.addFile(fpath);
+
+ if (qt.shouldBeSkipped(fname)) {
+ System.err.println("Test " + fname + " skipped");
+ return;
+ }
+
+ qt.cliInit(fname, false);
+ int ecode = qt.executeClient(fname);
+ if (ecode == 0) {
+ qt.failed(fname, debugHint);
+ }
+
+ ecode = qt.checkCliDriverResults(fname);
+ if (ecode != 0) {
+ qt.failedDiff(ecode, fname, debugHint);
+ }
+ }
+ catch (Throwable e) {
+ qt.failed(e, fname, debugHint);
+ }
+
+ long elapsedTime = System.currentTimeMillis() - startTime;
+ System.err.println("Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s");
+ assertTrue("Test passed", true);
+ }
+}
diff --git itests/pom.xml itests/pom.xml
index 426ba04..14dfad3 100644
--- itests/pom.xml
+++ itests/pom.xml
@@ -36,6 +36,7 @@
custom-udfs
hcatalog-unit
hive-unit
+ hive-blobstore
util
test-serde
qtest
diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index b43c4a7..704edbf 100644
--- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -1479,7 +1479,8 @@ private void maskPatterns(Pattern[] patterns, String fname) throws Exception {
"fk_-?[0-9]*_[0-9]*_[0-9]*",
".*at com\\.sun\\.proxy.*",
".*at com\\.jolbox.*",
- "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*"
+ "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*",
+ "(s3.?|swift|wasb.?):\\/\\/[\\w\\.\\/-]*"
});
private final Pattern[] partialReservedPlanMask = toPattern(new String[] {
@@ -2158,7 +2159,7 @@ public int compare(String str1, String str2) {
int i = 0;
while (!statements.isEmpty()) {
// PreparedStatement extend Statement
- Statement st = (Statement)statements.remove(i);
+ Statement st = statements.remove(i);
try {
if (st != null) {
st.close();