commit 47f45705f997658b3ce5df4276915a6c0be56d8a Author: Steve Yeom Date: Fri Feb 2 16:48:02 2018 -0800 HIVE-18599: Transactions: Fix CTAS on Micromanaged tables (Steve Yeom, reviewed by Gopal V) Signed-off-by: Gopal V diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 70d0749..fed9394 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -71,6 +71,7 @@ minillap.shared.query.files=insert_into1.q,\ mapreduce1.q,\ mapreduce2.q,\ mm_all.q,\ + mm_cttas.q,\ orc_merge1.q,\ orc_merge10.q,\ orc_merge2.q,\ diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index dbf9363..c2e2499 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2053,8 +2053,15 @@ private void getMetaData(QB qb, ReadEntity parentInput) } } try { - fname = ctx.getExtTmpPathRelTo( - FileUtils.makeQualified(location, conf)).toString(); + CreateTableDesc tblDesc = qb.getTableDesc(); + if (tblDesc != null + && tblDesc.isTemporary() + && AcidUtils.isInsertOnlyTable(tblDesc.getTblProps(), true)) { + fname = FileUtils.makeQualified(location, conf).toString(); + } else { + fname = ctx.getExtTmpPathRelTo( + FileUtils.makeQualified(location, conf)).toString(); + } } catch (Exception e) { throw new SemanticException(generateErrorMessage(ast, "Error creating temporary folder on: " + location.toString()), e); @@ -6842,7 +6849,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) field_schemas = new ArrayList(); destTableIsTemporary = tblDesc.isTemporary(); destTableIsMaterialization = tblDesc.isMaterialization(); - if (!destTableIsTemporary && AcidUtils.isInsertOnlyTable(tblDesc.getTblProps(), true)) { + if (AcidUtils.isInsertOnlyTable(tblDesc.getTblProps(), true)) { isMmTable = isMmCtas = true; txnId = SessionState.get().getTxnMgr().getCurrentTxnId(); tblDesc.setInitialMmWriteId(txnId); diff --git ql/src/test/queries/clientpositive/mm_cttas.q ql/src/test/queries/clientpositive/mm_cttas.q new file mode 100644 index 0000000..b099d2f --- /dev/null +++ ql/src/test/queries/clientpositive/mm_cttas.q @@ -0,0 +1,20 @@ + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +drop table intermediate; +create table intermediate(key int) partitioned by (p int) stored as orc; +insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2; +insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2; +insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2; + +drop table cttas1_mm; + +create temporary table cttas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate; + +select * from cttas1_mm; + +drop table cttas1_mm; + +drop table intermediate; + diff --git ql/src/test/results/clientpositive/llap/mm_cttas.q.out ql/src/test/results/clientpositive/llap/mm_cttas.q.out new file mode 100644 index 0000000..01acec5 --- /dev/null +++ ql/src/test/results/clientpositive/llap/mm_cttas.q.out @@ -0,0 +1,89 @@ +PREHOOK: query: drop table intermediate +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table intermediate +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@intermediate +POSTHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@intermediate +PREHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@intermediate@p=455 +POSTHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@intermediate@p=455 +POSTHOOK: Lineage: intermediate PARTITION(p=455).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@intermediate@p=456 +POSTHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@intermediate@p=456 +POSTHOOK: Lineage: intermediate PARTITION(p=456).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@intermediate@p=457 +POSTHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@intermediate@p=457 +POSTHOOK: Lineage: intermediate PARTITION(p=457).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: drop table cttas1_mm +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table cttas1_mm +POSTHOOK: type: DROPTABLE +PREHOOK: query: create temporary table cttas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@intermediate +PREHOOK: Input: default@intermediate@p=455 +PREHOOK: Input: default@intermediate@p=456 +PREHOOK: Input: default@intermediate@p=457 +PREHOOK: Output: database:default +PREHOOK: Output: default@cttas1_mm +POSTHOOK: query: create temporary table cttas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@intermediate +POSTHOOK: Input: default@intermediate@p=455 +POSTHOOK: Input: default@intermediate@p=456 +POSTHOOK: Input: default@intermediate@p=457 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cttas1_mm +PREHOOK: query: select * from cttas1_mm +PREHOOK: type: QUERY +PREHOOK: Input: default@cttas1_mm +#### A masked pattern was here #### +POSTHOOK: query: select * from cttas1_mm +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cttas1_mm +#### A masked pattern was here #### +98 455 +97 455 +100 457 +103 457 +0 456 +10 456 +PREHOOK: query: drop table cttas1_mm +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@cttas1_mm +PREHOOK: Output: default@cttas1_mm +POSTHOOK: query: drop table cttas1_mm +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@cttas1_mm +POSTHOOK: Output: default@cttas1_mm +PREHOOK: query: drop table intermediate +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@intermediate +PREHOOK: Output: default@intermediate +POSTHOOK: query: drop table intermediate +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@intermediate +POSTHOOK: Output: default@intermediate