diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index abeb7fc9cf..5155820508 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -71,6 +71,7 @@ import org.apache.hadoop.hive.ql.cache.results.CacheUsage; import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache; import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache.CacheEntry; +import org.apache.hadoop.hive.ql.exec.AbstractFileMergeOperator; import org.apache.hadoop.hive.ql.exec.ConditionalTask; import org.apache.hadoop.hive.ql.exec.DagUtils; import org.apache.hadoop.hive.ql.exec.ExplainTask; @@ -1579,9 +1580,18 @@ private void acquireLocks() throws CommandProcessorResponse { Utilities.getTableName(tableInfo.getTableName())); desc.setTableWriteId(writeId); - //it's possible to have > 1 FileSink writing to the same table/partition - //e.g. Merge stmt, multi-insert stmt when mixing DP and SP writes + /** + * it's possible to have > 1 FileSink writing to the same table/partition + * e.g. Merge stmt, multi-insert stmt when mixing DP and SP writes + * Insert ... Select ... Union All Select ... using + * {@link org.apache.hadoop.hive.ql.exec.AbstractFileMergeOperator#UNION_SUDBIR_PREFIX} + */ desc.setStatementId(queryTxnMgr.getStmtIdAndIncrement()); + String unionAllSubdir = "/" + AbstractFileMergeOperator.UNION_SUDBIR_PREFIX; + if(desc.getInsertOverwrite() && desc.getDirName().toString().contains(unionAllSubdir)) { + throw new UnsupportedOperationException("QueryId=" + plan.getQueryId() + + " is not supported due to OVERWRITE and UNION ALL. Please use truncate + insert"); + } } } diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java index f071531cec..7ab76b3437 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java @@ -275,6 +275,24 @@ public void testInsertToAcidWithUnionRemove() throws Exception { }; checkExpected(rs, expected, "Unexpected row count after ctas"); } + @Test + public void testInsertOverwriteToAcidWithUnionRemove() throws Exception { + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_UNION_REMOVE, true); + hiveConf.setVar(HiveConf.ConfVars.HIVEFETCHTASKCONVERSION, "none"); + d.close(); + d = new Driver(hiveConf); + int[][] values = {{1, 2}, {3, 4}, {5, 6}, {7, 8}, {9, 10}}; + runStatementOnDriver("drop table if exists T"); + runStatementOnDriver("create table T (a int, b int) stored as ORC TBLPROPERTIES ('transactional'='true')"); + + CommandProcessorResponse cpr = runStatementOnDriverNegative( + "insert overwrite table T select a, b from " + TxnCommandsBaseForTests.Table.ACIDTBL + + " where a between 1 and 3 group by a, b union all select a, b from " + + TxnCommandsBaseForTests.Table.ACIDTBL + + " where a between 5 and 7 union all select a, b from " + + TxnCommandsBaseForTests.Table.ACIDTBL + " where a >= 9"); + Assert.assertTrue("", cpr.getErrorMessage().contains("not supported due to OVERWRITE and UNION ALL")); + } /** * The idea here is to create a non acid table that was written by multiple writers, i.e. * unbucketed table that has 000000_0 & 000001_0, for example.