diff --git pom.xml pom.xml index c2e9ac3..b1a6406 100644 --- pom.xml +++ pom.xml @@ -100,7 +100,7 @@ 2.4 2.4 2.4.3 - 2.19.1 + 2.18.1 2.4 2.8 2.9 diff --git ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java index 28ca77c..f7d46ee 100644 --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java @@ -260,11 +260,21 @@ LockState acquireLocks(QueryPlan plan, Context ctx, String username, boolean isB LOG.debug("output is null " + (output == null)); switch (output.getWriteType()) { case DDL_EXCLUSIVE: - case INSERT_OVERWRITE: compBuilder.setExclusive(); compBuilder.setOperationType(DataOperationType.NO_TXN); break; - + case INSERT_OVERWRITE: + t = getTable(output); + if(AcidUtils.isAcidTable(t)) { + compBuilder.setSemiShared(); + //todo: don't know if this will have the right semantics under concurrent other writes + compBuilder.setOperationType(DataOperationType.UPDATE); + } + else { + compBuilder.setExclusive(); + compBuilder.setOperationType(DataOperationType.NO_TXN); + } + break; case INSERT: t = getTable(output); if(AcidUtils.isAcidTable(t)) { diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index c5b3517..22c5dbe 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -2011,7 +2011,7 @@ public void loadTable(Path loadPath, String tableName, boolean replace, boolean if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary()) { newFiles = Collections.synchronizedList(new ArrayList()); } - if (replace) { + if (replace && !isAcid) { Path tableDest = tbl.getPath(); replaceFiles(tableDest, loadPath, tableDest, tableDest, sessionConf, isSrcLocal); } else { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index f275f6a..2c7fc32 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -7217,7 +7217,16 @@ private void checkAcidConstraints(QB qb, TableDesc tableDesc, String tableName = tableDesc.getTableName(); if (!qb.getParseInfo().isInsertIntoTable(tableName)) { LOG.debug("Couldn't find table " + tableName + " in insertIntoTable"); - throw new SemanticException(ErrorMsg.NO_INSERT_OVERWRITE_WITH_ACID.getMsg()); + boolean isInsertOverwrite = false; + for(String key: qb.getParseInfo().getInsertOverwriteTables().keySet()) { + if(key.equalsIgnoreCase(tableName)) { + isInsertOverwrite = true; + break; + } + } + if(!isInsertOverwrite) { + throw new SemanticException(ErrorMsg.NO_INSERT_OVERWRITE_WITH_ACID.getMsg()); + } } /* LOG.info("Modifying config values for ACID write"); diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java index 9bfcc82..fb88251 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.processors.CommandProcessor; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService; @@ -73,7 +74,7 @@ ).getPath().replaceAll("\\\\", "/"); private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; //bucket count for test tables; set it to 1 for easier debugging - private static int BUCKET_COUNT = 2; + private static int BUCKET_COUNT = 1; @Rule public TestName testName = new TestName(); private HiveConf hiveConf; @@ -144,12 +145,6 @@ public void tearDown() throws Exception { FileUtils.deleteDirectory(new File(TEST_DATA_DIR)); } } - @Test - public void testInsertOverwrite() throws Exception { - runStatementOnDriver("insert overwrite table " + Table.NONACIDORCTBL + " select a,b from " + Table.NONACIDORCTBL2); - runStatementOnDriver("create table " + Table.NONACIDORCTBL2 + "3(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='false')"); - - } @Ignore("not needed but useful for testing") @Test public void testNonAcidInsert() throws Exception { @@ -802,4 +797,10 @@ public void testSetClauseFakeColumn() throws Exception { Assert.assertEquals(ErrorMsg.INVALID_TARGET_COLUMN_IN_SET_CLAUSE, ((HiveException)cpr.getException()).getCanonicalErrorMsg()); } + @Test + public void testInsertOverwrite() throws Exception { +// runStatementOnDriver("insert overwrite table " + Table.NONACIDORCTBL + " select * from " + Table.ACIDTBL); + runStatementOnDriver("insert into " + Table.NONACIDORCTBL + " values(1,2)"); + runStatementOnDriver("insert overwrite table " + Table.ACIDTBL + " select * from " + Table.NONACIDORCTBL); + } }