diff --git data/conf/hive-site.xml data/conf/hive-site.xml
index 9f6aec178d..b91bf86b7d 100644
--- data/conf/hive-site.xml
+++ data/conf/hive-site.xml
@@ -69,6 +69,21 @@
+ metastore.create.as.acid
+ true
+
+
+
+ hive.exec.dynamic.partition.mode
+ nonstrict
+
+
+
javax.jdo.option.ConnectionURL
jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db;create=true
diff --git data/conf/llap/hive-site.xml data/conf/llap/hive-site.xml
index cdda875ddb..e64fc9146a 100644
--- data/conf/llap/hive-site.xml
+++ data/conf/llap/hive-site.xml
@@ -166,10 +166,22 @@
org.apache.hadoop.hive.ql.hooks.PostExecutePrinter
Post Execute Hook for Tests
+
+ metastore.create.as.acid
+ true
+
+
+ hive.txn.manager
+ org.apache.hadoop.hive.ql.lockmgr.DbTxnManager
+
+
+ hive.exec.dynamic.partition.mode
+ nonstrict
+
hive.support.concurrency
- false
+ true
Whether hive supports concurrency or not. A zookeeper instance must be up and running for the default hive lock manager to support read-write locks.
diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java
index 616391f962..0d3d6cb731 100644
--- hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java
+++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java
@@ -80,7 +80,6 @@ protected void setUpHiveConf() {
hiveConf = new HiveConf(this.getClass());
hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, "");
hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, "");
- hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, TEST_WAREHOUSE_DIR);
hiveConf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
hiveConf.setBoolVar(HiveConf.ConfVars.HIVEOPTIMIZEMETADATAQUERIES, true);
diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
index ba42ffd627..26d5b0f66e 100644
--- hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
+++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
@@ -189,7 +189,7 @@ private static void initializeSetup() throws Exception {
HCatSemanticAnalyzer.class.getName());
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
- hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+// hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " ");
System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " ");
diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java
index 6f372c979a..1fb19b1e1a 100644
--- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java
+++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java
@@ -87,7 +87,6 @@ public void setUp() throws Exception {
HiveConf hiveConf = new HiveConf(this.getClass());
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
- hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);
hiveConf
.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
index 5c6626d84d..9c61a46b1c 100644
--- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
+++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
@@ -110,7 +110,6 @@ public static void setUpBeforeClass() throws Exception {
HiveConf hiveConf = new HiveConf(TestHCatLoaderComplexSchema.class);
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
- hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
hiveConf
.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
"org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
index d1f46e4af4..644dea16c0 100644
--- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
+++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderEncryption.java
@@ -164,7 +164,6 @@ public void setup() throws Exception {
HiveConf hiveConf = new HiveConf(this.getClass());
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
- hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);
hiveConf
.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
diff --git hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
index c402ba8e6a..4d1ee76b10 100644
--- hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
+++ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
@@ -99,7 +99,6 @@ public void setUp() throws Exception {
HiveConf hiveConf = new HiveConf(this.getClass());
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
- hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);
hiveConf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
hiveConf
diff --git hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
index 78e767e7fc..d36abe0c66 100644
--- hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
+++ hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
@@ -130,6 +130,13 @@ public static void startMetaStoreServer() throws Exception {
hcatConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
hcatConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname,
"false");
+ /*All HCat tests (and those that depend on them) are built with the expectation that
+ * hive.support.concurrency=false. See line above. HIVE_SUPPORT_CONCURRENCY and
+ * HIVE_TXN_MANAGER are related in that DbTxnManager requires concurrency=true. Unfortunately,
+ * JUnit tests don't use hive-site.xml like various forms of CLI driver tests. So I'm setting
+ * txn.manager here explicitly as well*/
+ hcatConf.set(HiveConf.ConfVars.HIVE_TXN_MANAGER.varname,
+ "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager");
System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " ");
System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " ");
}
diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
index 2bb06d582c..8094ca4f67 100644
--- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
+++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
@@ -149,7 +149,7 @@ public static void setUpBeforeClass() throws Exception {
hconf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
hconf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hconf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
- hconf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+// hconf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
hconf.set(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL.varname,
"org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore");
hconf.setBoolVar(HiveConf.ConfVars.HIVEOPTIMIZEMETADATAQUERIES, true);
diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/BeelineWithHS2ConnectionFileTestBase.java itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/BeelineWithHS2ConnectionFileTestBase.java
index 2fb64536a6..4b25773ffd 100644
--- itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/BeelineWithHS2ConnectionFileTestBase.java
+++ itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/BeelineWithHS2ConnectionFileTestBase.java
@@ -161,7 +161,7 @@ public void before() throws Exception {
hiveConf = new HiveConf();
miniHS2 = getNewMiniHS2();
confOverlay = new HashMap();
- confOverlay.put(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+// confOverlay.put(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, "binary");
}
diff --git itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java
index 830ffc2bbd..b51bca8a84 100644
--- itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java
+++ itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java
@@ -66,7 +66,6 @@ public static void setUpBeforeClass() throws Exception {
hiveConf.set(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "verbose");
miniHS2 = new MiniHS2(hiveConf);
confOverlay = new HashMap();
- confOverlay.put(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
miniHS2.start(confOverlay);
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 022ba04fbe..c4c12bbb6f 100644
--- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -3376,16 +3376,30 @@ private static Path mvFile(HiveConf conf, FileSystem sourceFs, Path sourcePath,
final String fullname = sourcePath.getName();
final String name = FilenameUtils.getBaseName(sourcePath.getName());
final String type = FilenameUtils.getExtension(sourcePath.getName());
+ final boolean isAcidTarget = destDirPath.getName().startsWith(AcidUtils.BASE_PREFIX) || destDirPath.getName().startsWith(AcidUtils.DELTA_PREFIX);
+ Path destFilePath;
+ if(isAcidTarget && !AcidUtils.originalBucketFilter.accept(sourcePath)) {
+ //if here we are doing a load Data into acid table - todo: make this more explicit
+ //Acid tables can only deal with files matching AcidUtils.originalBucketFilter.
+ //so here we rename the input file and further logic will add a copy_N suffix in case of
+ //collisions. (This works since Load Data doesn't support bucketed tables for now)
+ destFilePath = new Path(destDirPath, "000000_0");
+ }
+ else {
+ destFilePath = new Path(destDirPath, fullname);
+ }
- Path destFilePath = new Path(destDirPath, fullname);
-
- /*
+ /*
* The below loop may perform bad when the destination file already exists and it has too many _copy_
* files as well. A desired approach was to call listFiles() and get a complete list of files from
* the destination, and check whether the file exists or not on that list. However, millions of files
* could live on the destination directory, and on concurrent situations, this can cause OOM problems.
*
* I'll leave the below loop for now until a better approach is found.
+ *
+ * This is problematic: caller of mvFile() may use a thread pool to move files in parallel in
+ * which case there is a race condition between exists() and reaname() from different threads.
+ * I suppose in case of collisions the FileSystem will throw and the command will fail.
*/
for (int counter = 1; destFs.exists(destFilePath); counter++) {
if (isOverwrite) {
diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index 4535c3edc2..ea63016ad4 100644
--- ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@ -160,7 +160,7 @@ private URI initializeFromURI(String fromPath, boolean isLocal) throws IOExcepti
"source contains directory: " + oneSrc.getPath().toString()));
}
if(AcidUtils.isAcidTable(table)) {
- if(!AcidUtils.originalBucketFilter.accept(oneSrc.getPath())) {
+ if(false && !AcidUtils.originalBucketFilter.accept(oneSrc.getPath())) {
//acid files (e.g. bucket_0000) have ROW_ID embedded in them and so can't be simply
//copied to a table so only allow non-acid files for now
throw new SemanticException(ErrorMsg.ACID_LOAD_DATA_INVALID_FILE_NAME,
diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
index c4911bbe7c..25839e1cd5 100644
--- ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
+++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.hive.ql;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.junit.Assert;
@@ -460,4 +463,64 @@ private void checkResult(String[][] expectedResult, String query, boolean isVect
checkExpected(rs, expectedResult, msg + (isVectorized ? " vect" : ""), LOG, !isVectorized);
assertVectorized(isVectorized, query);
}
+ @Test
+ public void testAnyFileName() throws Exception {
+ boolean isVectorized = false;
+ runStatementOnDriver("drop table if exists T");
+ runStatementOnDriver("create table T ( ctinyint TINYINT,\n" +
+ " csmallint SMALLINT,\n" +
+ " cint INT,\n" +
+ " cbigint BIGINT,\n" +
+ " cfloat FLOAT,\n" +
+ " cdouble DOUBLE,\n" +
+ " cstring1 STRING,\n" +
+ " cstring2 STRING,\n" +
+ " ctimestamp1 TIMESTAMP,\n" +
+ " ctimestamp2 TIMESTAMP,\n" +
+ " cboolean1 BOOLEAN,\n" +
+ " cboolean2 BOOLEAN) stored as orc tblproperties('transactional'='true')");
+ //ql/target/tmp/org.apache.hadoop.hive.ql.TestTxnNoBuckets-1512791382683/warehouse
+ runStatementOnDriver("load data local inpath '" + getWarehouseDir() + "/../../../../../data/files/alltypesorc' into table T");
+ List rs = runStatementOnDriver("select count(*) from T");
+ Assert.assertEquals("12288", rs.get(0));
+ runStatementOnDriver("load data local inpath '" + getWarehouseDir() + "/../../../../../data/files/alltypesorc' into table T");
+ rs = runStatementOnDriver("select count(*) from T");
+ Assert.assertEquals("24576", rs.get(0));
+ String[][] expected = new String[][] {
+ {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":0}", "t/delta_0000015_0000015_0000/000000_0"},
+ {"{\"transactionid\":17,\"bucketid\":536870912,\"rowid\":0}", "t/delta_0000017_0000017_0000/000000_0"}
+ };
+ checkResult(expected, "select ROW__ID, INPUT__FILE__NAME from T where ROW__ID.rowid=0 limit 2", isVectorized, "load data inpath");
+ }
+ @Test
+ public void testAnyFileName2() throws Exception {
+ boolean isVectorized = false;
+ runStatementOnDriver("drop table if exists T");
+ runStatementOnDriver("drop table if exists Tstage");
+ runStatementOnDriver("create table Tstage (a int, b int) clustered by (a) into 2 buckets stored as orc");
+ runStatementOnDriver("insert into Tstage values(1,2),(2,2)");
+ runStatementOnDriver("export table Tstage to '" + getWarehouseDir() + "/1'");
+ runStatementOnDriver("create table T (a int, b int) stored as orc tblproperties('transactional'='true')");
+ FileSystem fs = FileSystem.get(hiveConf);
+ FileStatus[] status;
+ status = fs.listStatus(new Path(getWarehouseDir() + "/1/data"),
+ org.apache.hadoop.hive.common.FileUtils.STAGING_DIR_PATH_FILTER);
+ fs.rename(status[0].getPath(), new Path(status[0].getPath().getParent(), "a_0"));
+ fs.rename(status[1].getPath(), new Path(status[1].getPath().getParent(), "b_0"));
+
+ runStatementOnDriver("load data local inpath '" + getWarehouseDir() + "/1/data' into table T");//only get 1 file in T
+
+ //ql/target/tmp/org.apache.hadoop.hive.ql.TestTxnNoBuckets-1512791382683/warehouse
+ runStatementOnDriver("load data local inpath '" + getWarehouseDir() + "/../../../../../data/files/alltypesorc' into table T");
+ List rs = runStatementOnDriver("select count(*) from T");
+ Assert.assertEquals("12288", rs.get(0));
+ runStatementOnDriver("load data local inpath '" + getWarehouseDir() + "/../../../../../data/files/alltypesorc' into table T");
+ rs = runStatementOnDriver("select count(*) from T");
+ Assert.assertEquals("24576", rs.get(0));
+ String[][] expected = new String[][] {
+ {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":0}", "t/delta_0000015_0000015_0000/000000_0"},
+ {"{\"transactionid\":17,\"bucketid\":536870912,\"rowid\":0}", "t/delta_0000017_0000017_0000/000000_0"}
+ };
+ checkResult(expected, "select ROW__ID, INPUT__FILE__NAME from T where ROW__ID.rowid=0 limit 2", isVectorized, "load data inpath");
+ }
}