diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index f4bd0f9399..75a0ea5d19 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -2997,9 +2997,9 @@ private void constructOneLBLocationMap(FileStatus fSta, .collect(Collectors.toList()), tableSnapshot); } catch (InterruptedException | ExecutionException e) { - throw new HiveException("Exception when loading " + validPartitions.size() + throw new HiveException("Exception when loading " + validPartitions.size() + " partitions" + " in table " + tbl.getTableName() - + " with loadPath=" + loadPath); + + " with loadPath=" + loadPath, e); } catch (TException e) { LOG.error(StringUtils.stringifyException(e)); throw new HiveException(e); @@ -3024,7 +3024,7 @@ private void constructOneLBLocationMap(FileStatus fSta, } try { - if (isAcid) { + if (isTxnTable) { List partNames = result.values().stream().map(Partition::getName).collect(Collectors.toList()); getMSC().addDynamicPartitions(parentSession.getTxnMgr().getCurrentTxnId(), writeId, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 90549f9f3a..c256702753 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -6884,8 +6884,9 @@ private Operator genBucketingSortingDest(String dest, Operator input, QB qb, nullOrder.append(sortOrder == DirectionUtils.ASCENDING_CODE ? 'a' : 'z'); } input = genReduceSinkPlan(input, partnCols, sortCols, order.toString(), nullOrder.toString(), - maxReducers, (AcidUtils.isFullAcidTable(dest_tab) ? - getAcidType(table_desc.getOutputFileFormatClass(), dest) : AcidUtils.Operation.NOT_ACID)); + maxReducers, + (AcidUtils.isFullAcidTable(dest_tab) ? getAcidType(table_desc.getOutputFileFormatClass(), + dest, AcidUtils.isInsertOnlyTable(dest_tab)) : AcidUtils.Operation.NOT_ACID)); reduceSinkOperatorsAddedByEnforceBucketingSorting.add((ReduceSinkOperator)input.getParentOperators().get(0)); ctx.setMultiFileSpray(multiFileSpray); ctx.setNumFiles(numFiles); @@ -7408,9 +7409,8 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // NOTE: specify Dynamic partitions in dest_tab for WriteEntity if (!isNonNativeTable) { AcidUtils.Operation acidOp = AcidUtils.Operation.NOT_ACID; - if (destTableIsFullAcid) { - acidOp = getAcidType(tableDescriptor.getOutputFileFormatClass(), dest); - //todo: should this be done for MM? is it ok to use CombineHiveInputFormat with MM + if (destTableIsTransactional) { + acidOp = getAcidType(tableDescriptor.getOutputFileFormatClass(), dest, isMmTable); checkAcidConstraints(); } try { @@ -7530,9 +7530,8 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) destinationPartition.getSkewedColValues(), destinationPartition.getSkewedColValueLocationMaps(), destinationPartition.isStoredAsSubDirectories()); AcidUtils.Operation acidOp = AcidUtils.Operation.NOT_ACID; - if (destTableIsFullAcid) { - acidOp = getAcidType(tableDescriptor.getOutputFileFormatClass(), dest); - //todo: should this be done for MM? is it ok to use CombineHiveInputFormat with MM? + if (destTableIsTransactional) { + acidOp = getAcidType(tableDescriptor.getOutputFileFormatClass(), dest, isMmTable); checkAcidConstraints(); } try { @@ -7619,7 +7618,8 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) tblProps = viewDesc.getTblProps(); } - if (tblProps != null && AcidUtils.isTablePropertyTransactional(tblProps)) { + destTableIsTransactional = tblProps != null && AcidUtils.isTablePropertyTransactional(tblProps); + if (destTableIsTransactional) { try { if (ctx.getExplainConfig() != null) { writeId = 0L; // For explain plan, txn won't be opened and doesn't make sense to allocate write id @@ -7789,9 +7789,8 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) boolean isNonNativeTable = tableDescriptor.isNonNative(); if (!isNonNativeTable) { AcidUtils.Operation acidOp = AcidUtils.Operation.NOT_ACID; - if (destTableIsFullAcid) { - acidOp = getAcidType(tableDescriptor.getOutputFileFormatClass(), dest); - //todo: should this be done for MM? is it ok to use CombineHiveInputFormat with MM + if (destTableIsTransactional) { + acidOp = getAcidType(tableDescriptor.getOutputFileFormatClass(), dest, isMmTable); checkAcidConstraints(); } // isReplace = false in case concurrent operation is executed @@ -8838,8 +8837,9 @@ private Operator genReduceSinkPlan(String dest, QB qb, Operator input, Table dest_tab = qb.getMetaData().getDestTableForAlias(dest); AcidUtils.Operation acidOp = Operation.NOT_ACID; - if (AcidUtils.isFullAcidTable(dest_tab)) { - acidOp = getAcidType(Utilities.getTableDesc(dest_tab).getOutputFileFormatClass(), dest); + if (AcidUtils.isTransactionalTable(dest_tab)) { + acidOp = getAcidType(Utilities.getTableDesc(dest_tab).getOutputFileFormatClass(), dest, + AcidUtils.isInsertOnlyTable(dest_tab)); } Operator result = genReduceSinkPlan( input, partCols, sortCols, order.toString(), nullOrder.toString(), @@ -14906,7 +14906,14 @@ private boolean isAcidOutputFormat(Class of) { AcidUtils.Operation.INSERT); } - private AcidUtils.Operation getAcidType(Class of, String dest) { + private AcidUtils.Operation getAcidType(Class of, String dest, + boolean isMM) { + + // no need for any checks in the case of insert-only + if (isMM) { + return getAcidType(dest); + } + if (SessionState.get() == null || !getTxnMgr().supportsAcid()) { return AcidUtils.Operation.NOT_ACID; } else if (isAcidOutputFormat(of)) { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java index c033a94bfa..f30a48e3d6 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java @@ -1558,6 +1558,34 @@ public void testMultiInsert() throws Exception { } //todo: Concurrent insert/update of same partition - should pass + @Test public void testMultiInsertOnDynamicallyPartitionedMmTable() throws Exception { + dropTable(new String[] {"tabMmDp", "tab_not_acid"}); + + driver.run("create table if not exists tabMmDp (a int, b int) partitioned by (p string) " + + "stored as orc " + + "TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only')"); + driver.run("create table if not exists tab_not_acid (a int, b int, p string)"); + driver.run("insert into tab_not_acid values (1 ,1, 'one'), (2, 2, 'two')"); + // insert 2 rows twice into the MM table + driver.run("from tab_not_acid " + + "insert into tabMmDp select a,b,p " + + "insert into tabMmDp select a,b,p"); //txnid: 6 (2 drops, 2 creates, 2 inserts) + + final String completedTxnComponentsContents = + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"); + Assert.assertEquals(completedTxnComponentsContents, + 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS")); + Assert.assertEquals(completedTxnComponentsContents, + 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=6")); + Assert.assertEquals(completedTxnComponentsContents, + 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=6 " + + "and ctc_table='tabmmdp'")); + // ctc_update_delete value should be "N" for both partitions since these are inserts + Assert.assertEquals(completedTxnComponentsContents, + 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=6 " + + "and ctc_table='tabmmdp' and ctc_update_delete='N'")); + } + private List getLocksWithFilterOptions(HiveTxnManager txnMgr, String dbName, String tblName, Map partSpec) throws Exception { if (dbName == null && tblName != null) { diff --git a/ql/src/test/queries/clientpositive/mm_all.q b/ql/src/test/queries/clientpositive/mm_all.q index 9b4d7b67ca..d7c1879838 100644 --- a/ql/src/test/queries/clientpositive/mm_all.q +++ b/ql/src/test/queries/clientpositive/mm_all.q @@ -2,6 +2,7 @@ --! qt:dataset:src -- MASK_LINEAGE +-- SORT_QUERY_RESULTS set hive.metastore.dml.events=true; set hive.mapred.mode=nonstrict; diff --git a/ql/src/test/results/clientpositive/llap/check_constraint.q.out b/ql/src/test/results/clientpositive/llap/check_constraint.q.out index 9f2c9a1cd0..1935dcb75d 100644 --- a/ql/src/test/results/clientpositive/llap/check_constraint.q.out +++ b/ql/src/test/results/clientpositive/llap/check_constraint.q.out @@ -3508,6 +3508,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.part_mm + Write Type: INSERT micromanaged table: true PREHOOK: query: insert into table part_mm partition(key_mm=455) select key from src order by value desc limit 3 @@ -3725,6 +3726,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tbl1_n1 + Write Type: INSERT micromanaged table: true PREHOOK: query: insert into tbl1_n1 values('a', 69) @@ -3857,6 +3859,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tbl1_n1 + Write Type: INSERT micromanaged table: true PREHOOK: query: drop table tbl1_n1 diff --git a/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out b/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out index 9343e078b7..80279a132d 100644 --- a/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out +++ b/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out @@ -6140,6 +6140,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.part_mm_n1 + Write Type: INSERT micromanaged table: true Stage: Stage-3 diff --git a/ql/src/test/results/clientpositive/llap/insert_only_empty_query.q.out b/ql/src/test/results/clientpositive/llap/insert_only_empty_query.q.out index 96bcd6faee..21a4f67560 100644 --- a/ql/src/test/results/clientpositive/llap/insert_only_empty_query.q.out +++ b/ql/src/test/results/clientpositive/llap/insert_only_empty_query.q.out @@ -148,6 +148,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.src_emptybucket_partitioned_1 + Write Type: INSERT micromanaged table: true Stage: Stage-3 diff --git a/ql/src/test/results/clientpositive/llap/mm_all.q.out b/ql/src/test/results/clientpositive/llap/mm_all.q.out index 7542a6ae2b..6e7c75211b 100644 --- a/ql/src/test/results/clientpositive/llap/mm_all.q.out +++ b/ql/src/test/results/clientpositive/llap/mm_all.q.out @@ -150,6 +150,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.part_mm_n0 + Write Type: INSERT micromanaged table: true Stage: Stage-3 @@ -223,18 +224,18 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 10 455 10 455 10 456 -97 455 -97 455 -97 456 -98 455 -98 455 -98 456 100 455 100 455 100 456 103 455 103 455 103 456 +97 455 +97 455 +97 456 +98 455 +98 455 +98 456 PREHOOK: query: select * from part_mm_n0 order by key, key_mm PREHOOK: type: QUERY PREHOOK: Input: default@part_mm_n0 @@ -253,18 +254,18 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 10 455 10 455 10 456 -97 455 -97 455 -97 456 -98 455 -98 455 -98 456 100 455 100 455 100 456 103 455 103 455 103 456 +97 455 +97 455 +97 456 +98 455 +98 455 +98 456 PREHOOK: query: truncate table part_mm_n0 PREHOOK: type: TRUNCATETABLE PREHOOK: Output: default@part_mm_n0@key_mm=455 @@ -330,10 +331,10 @@ POSTHOOK: Input: default@simple_mm POSTHOOK: Output: hdfs://### HDFS PATH ### 0 10 -97 -98 100 103 +97 +98 PREHOOK: query: insert into table simple_mm select key from intermediate_n0 PREHOOK: type: QUERY PREHOOK: Input: default@intermediate_n0 @@ -361,14 +362,14 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 0 10 10 -97 -97 -98 -98 100 100 103 103 +97 +97 +98 +98 PREHOOK: query: truncate table simple_mm PREHOOK: type: TRUNCATETABLE PREHOOK: Output: default@simple_mm @@ -452,10 +453,10 @@ POSTHOOK: Input: default@dp_mm@key1=123/key2=98 POSTHOOK: Output: hdfs://### HDFS PATH ### 0 123 0 10 123 10 -97 123 97 -98 123 98 100 123 100 103 123 103 +97 123 97 +98 123 98 PREHOOK: query: drop table dp_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@dp_mm @@ -506,15 +507,15 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 0 1 10 +100 +101 +103 +104 11 97 98 98 99 -100 -101 -103 -104 PREHOOK: query: insert into table union_mm select p from ( @@ -558,20 +559,8 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 0 1 1 -2 10 10 -11 -11 -12 -97 -97 -98 -98 -98 -99 -99 -99 100 100 100 @@ -583,6 +572,18 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 104 104 105 +11 +11 +12 +2 +97 +97 +98 +98 +98 +99 +99 +99 PREHOOK: query: insert into table union_mm SELECT p FROM ( @@ -642,27 +643,9 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 1 1 1 -2 -2 10 10 10 -11 -11 -11 -12 -12 -97 -97 -97 -98 -98 -98 -98 -99 -99 -99 -99 100 100 100 @@ -680,6 +663,24 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 104 105 105 +11 +11 +11 +12 +12 +2 +2 +97 +97 +97 +98 +98 +98 +98 +99 +99 +99 +99 PREHOOK: query: drop table union_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@union_mm @@ -772,15 +773,15 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 0 0 1 1 10 10 +100 100 +101 101 +103 103 +104 104 11 11 97 97 98 98 98 98 99 99 -100 100 -101 101 -103 103 -104 104 PREHOOK: query: drop table partunion_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@partunion_mm @@ -828,10 +829,10 @@ POSTHOOK: Input: default@skew_mm POSTHOOK: Output: hdfs://### HDFS PATH ### 0 0 0 10 10 10 -97 97 97 -98 98 98 100 100 100 103 103 103 +97 97 97 +98 98 98 PREHOOK: query: drop table skew_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@skew_mm @@ -952,15 +953,15 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 0 0 0 0 1 2 3 4 10 10 10 10 +100 100 100 100 +101 102 103 104 +103 103 103 103 +104 105 106 107 11 12 13 14 97 97 97 97 98 98 98 98 98 99 100 101 99 100 101 102 -100 100 100 100 -101 102 103 104 -103 103 103 103 -104 105 106 107 PREHOOK: query: drop table skew_dp_union_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@skew_dp_union_mm @@ -1000,12 +1001,12 @@ POSTHOOK: query: select * from merge0_mm POSTHOOK: type: QUERY POSTHOOK: Input: default@merge0_mm POSTHOOK: Output: hdfs://### HDFS PATH ### -98 -97 -100 -103 0 10 +100 +103 +97 +98 PREHOOK: query: insert into table merge0_mm select key from intermediate_n0 PREHOOK: type: QUERY PREHOOK: Input: default@intermediate_n0 @@ -1029,18 +1030,18 @@ POSTHOOK: query: select * from merge0_mm POSTHOOK: type: QUERY POSTHOOK: Input: default@merge0_mm POSTHOOK: Output: hdfs://### HDFS PATH ### -98 -97 -100 -103 +0 0 10 -98 -97 +10 +100 100 103 -0 -10 +103 +97 +97 +98 +98 PREHOOK: query: drop table merge0_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@merge0_mm @@ -1080,12 +1081,12 @@ POSTHOOK: query: select * from merge2_mm POSTHOOK: type: QUERY POSTHOOK: Input: default@merge2_mm POSTHOOK: Output: hdfs://### HDFS PATH ### -98 -97 -100 -103 0 10 +100 +103 +97 +98 PREHOOK: query: insert into table merge2_mm select key from intermediate_n0 PREHOOK: type: QUERY PREHOOK: Input: default@intermediate_n0 @@ -1109,18 +1110,18 @@ POSTHOOK: query: select * from merge2_mm POSTHOOK: type: QUERY POSTHOOK: Input: default@merge2_mm POSTHOOK: Output: hdfs://### HDFS PATH ### -98 -97 -100 -103 +0 0 10 -98 -97 +10 +100 100 103 -0 -10 +103 +97 +97 +98 +98 PREHOOK: query: drop table merge2_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@merge2_mm @@ -1184,10 +1185,10 @@ POSTHOOK: Input: default@merge1_mm@key=98 POSTHOOK: Output: hdfs://### HDFS PATH ### 0 0 10 10 -97 97 -98 98 100 100 103 103 +97 97 +98 98 PREHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate_n0 PREHOOK: type: QUERY PREHOOK: Input: default@intermediate_n0 @@ -1237,14 +1238,14 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 0 0 10 10 10 10 -97 97 -97 97 -98 98 -98 98 100 100 100 100 103 103 103 103 +97 97 +97 97 +98 98 +98 98 PREHOOK: query: drop table merge1_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@merge1_mm @@ -1283,12 +1284,12 @@ POSTHOOK: query: select * from ctas0_mm POSTHOOK: type: QUERY POSTHOOK: Input: default@ctas0_mm POSTHOOK: Output: hdfs://### HDFS PATH ### -98 455 -97 455 -100 457 -103 457 0 456 10 456 +100 457 +103 457 +97 455 +98 455 PREHOOK: query: drop table ctas0_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@ctas0_mm @@ -1329,18 +1330,18 @@ POSTHOOK: query: select * from ctas1_mm POSTHOOK: type: QUERY POSTHOOK: Input: default@ctas1_mm POSTHOOK: Output: hdfs://### HDFS PATH ### -98 455 -97 455 -100 457 -103 457 +0 456 0 456 10 456 -98 455 -97 455 +10 456 +100 457 100 457 103 457 -0 456 -10 456 +103 457 +97 455 +97 455 +98 455 +98 455 PREHOOK: query: drop table ctas1_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@ctas1_mm @@ -1407,10 +1408,10 @@ POSTHOOK: Input: default@multi0_1_mm POSTHOOK: Output: hdfs://### HDFS PATH ### 0 456 10 456 -97 455 -98 455 100 457 103 457 +97 455 +98 455 PREHOOK: query: select * from multi0_2_mm order by key, key2 PREHOOK: type: QUERY PREHOOK: Input: default@multi0_2_mm @@ -1459,8 +1460,6 @@ POSTHOOK: Input: default@multi0_1_mm POSTHOOK: Output: hdfs://### HDFS PATH ### 0 456 10 456 -97 455 -98 455 100 457 103 457 455 97 @@ -1469,6 +1468,8 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 456 10 457 100 457 103 +97 455 +98 455 PREHOOK: query: select * from multi0_2_mm order by key, key2 PREHOOK: type: QUERY PREHOOK: Input: default@multi0_2_mm @@ -1479,10 +1480,10 @@ POSTHOOK: Input: default@multi0_2_mm POSTHOOK: Output: hdfs://### HDFS PATH ### 0 456 10 456 -97 455 -98 455 100 457 103 457 +97 455 +98 455 PREHOOK: query: drop table multi0_1_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@multi0_1_mm @@ -1549,8 +1550,6 @@ POSTHOOK: Input: default@multi1_mm@p=2 POSTHOOK: Output: hdfs://### HDFS PATH ### 0 456 2 10 456 2 -97 455 2 -98 455 2 100 457 2 103 457 2 455 97 1 @@ -1559,6 +1558,8 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 456 10 1 457 100 1 457 103 1 +97 455 2 +98 455 2 PREHOOK: query: from intermediate_n0 insert into table multi1_mm partition(p=2) select p, key insert overwrite table multi1_mm partition(p=1) select key, p @@ -1599,10 +1600,6 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 0 456 2 10 456 1 10 456 2 -97 455 1 -97 455 2 -98 455 1 -98 455 2 100 457 1 100 457 2 103 457 1 @@ -1613,6 +1610,10 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 456 10 2 457 100 2 457 103 2 +97 455 1 +97 455 2 +98 455 1 +98 455 2 PREHOOK: query: from intermediate_n0 insert into table multi1_mm partition(p) select p, key, p insert into table multi1_mm partition(p=1) select key, p @@ -1669,12 +1670,6 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 10 456 1 10 456 1 10 456 2 -97 455 1 -97 455 1 -97 455 2 -98 455 1 -98 455 1 -98 455 2 100 457 1 100 457 1 100 457 2 @@ -1693,6 +1688,12 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 457 100 457 457 103 2 457 103 457 +97 455 1 +97 455 1 +97 455 2 +98 455 1 +98 455 1 +98 455 2 PREHOOK: query: from intermediate_n0 insert into table multi1_mm partition(p) select p, key, 1 insert into table multi1_mm partition(p=1) select key, p @@ -1742,14 +1743,6 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 10 456 1 10 456 1 10 456 2 -97 455 1 -97 455 1 -97 455 1 -97 455 2 -98 455 1 -98 455 1 -98 455 1 -98 455 2 100 457 1 100 457 1 100 457 1 @@ -1776,6 +1769,14 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 457 103 1 457 103 2 457 103 457 +97 455 1 +97 455 1 +97 455 1 +97 455 2 +98 455 1 +98 455 1 +98 455 1 +98 455 2 PREHOOK: query: drop table multi1_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@multi1_mm @@ -2122,12 +2123,12 @@ POSTHOOK: query: SELECT * FROM temp1 POSTHOOK: type: QUERY POSTHOOK: Input: default@temp1 POSTHOOK: Output: hdfs://### HDFS PATH ### -98 -97 -100 -103 0 10 +100 +103 +97 +98 PREHOOK: query: drop table intermediate_n0 PREHOOK: type: DROPTABLE PREHOOK: Input: default@intermediate_n0 diff --git a/ql/src/test/results/clientpositive/llap/mm_dp.q.out b/ql/src/test/results/clientpositive/llap/mm_dp.q.out index 94da4ff56f..212007700a 100644 --- a/ql/src/test/results/clientpositive/llap/mm_dp.q.out +++ b/ql/src/test/results/clientpositive/llap/mm_dp.q.out @@ -185,6 +185,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.dp_mm + Write Type: INSERT micromanaged table: true Stage: Stage-3 diff --git a/ql/src/test/results/clientpositive/mm_all.q.out b/ql/src/test/results/clientpositive/mm_all.q.out index 1377856495..8f1bb0fcf8 100644 --- a/ql/src/test/results/clientpositive/mm_all.q.out +++ b/ql/src/test/results/clientpositive/mm_all.q.out @@ -147,6 +147,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.part_mm_n0 + Write Type: INSERT micromanaged table: true Stage: Stage-2 @@ -237,18 +238,18 @@ POSTHOOK: Input: default@part_mm_n0@key_mm=456 10 455 10 455 10 456 -97 455 -97 455 -97 456 -98 455 -98 455 -98 456 100 455 100 455 100 456 103 455 103 455 103 456 +97 455 +97 455 +97 456 +98 455 +98 455 +98 456 PREHOOK: query: select * from part_mm_n0 order by key, key_mm PREHOOK: type: QUERY PREHOOK: Input: default@part_mm_n0 @@ -267,18 +268,18 @@ POSTHOOK: Input: default@part_mm_n0@key_mm=456 10 455 10 455 10 456 -97 455 -97 455 -97 456 -98 455 -98 455 -98 456 100 455 100 455 100 456 103 455 103 455 103 456 +97 455 +97 455 +97 456 +98 455 +98 455 +98 456 PREHOOK: query: truncate table part_mm_n0 PREHOOK: type: TRUNCATETABLE PREHOOK: Output: default@part_mm_n0@key_mm=455 @@ -344,10 +345,10 @@ POSTHOOK: Input: default@simple_mm #### A masked pattern was here #### 0 10 -97 -98 100 103 +97 +98 PREHOOK: query: insert into table simple_mm select key from intermediate_n0 PREHOOK: type: QUERY PREHOOK: Input: default@intermediate_n0 @@ -375,14 +376,14 @@ POSTHOOK: Input: default@simple_mm 0 10 10 -97 -97 -98 -98 100 100 103 103 +97 +97 +98 +98 PREHOOK: query: truncate table simple_mm PREHOOK: type: TRUNCATETABLE PREHOOK: Output: default@simple_mm @@ -466,10 +467,10 @@ POSTHOOK: Input: default@dp_mm@key1=123/key2=98 #### A masked pattern was here #### 0 123 0 10 123 10 -97 123 97 -98 123 98 100 123 100 103 123 103 +97 123 97 +98 123 98 PREHOOK: query: drop table dp_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@dp_mm @@ -520,15 +521,15 @@ POSTHOOK: Input: default@union_mm 0 1 10 +100 +101 +103 +104 11 97 98 98 99 -100 -101 -103 -104 PREHOOK: query: insert into table union_mm select p from ( @@ -572,20 +573,8 @@ POSTHOOK: Input: default@union_mm 0 1 1 -2 10 10 -11 -11 -12 -97 -97 -98 -98 -98 -99 -99 -99 100 100 100 @@ -597,6 +586,18 @@ POSTHOOK: Input: default@union_mm 104 104 105 +11 +11 +12 +2 +97 +97 +98 +98 +98 +99 +99 +99 PREHOOK: query: insert into table union_mm SELECT p FROM ( @@ -656,27 +657,9 @@ POSTHOOK: Input: default@union_mm 1 1 1 -2 -2 10 10 10 -11 -11 -11 -12 -12 -97 -97 -97 -98 -98 -98 -98 -99 -99 -99 -99 100 100 100 @@ -694,6 +677,24 @@ POSTHOOK: Input: default@union_mm 104 105 105 +11 +11 +11 +12 +12 +2 +2 +97 +97 +97 +98 +98 +98 +98 +99 +99 +99 +99 PREHOOK: query: drop table union_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@union_mm @@ -786,15 +787,15 @@ POSTHOOK: Input: default@partunion_mm@key=99 0 0 1 1 10 10 +100 100 +101 101 +103 103 +104 104 11 11 97 97 98 98 98 98 99 99 -100 100 -101 101 -103 103 -104 104 PREHOOK: query: drop table partunion_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@partunion_mm @@ -842,10 +843,10 @@ POSTHOOK: Input: default@skew_mm #### A masked pattern was here #### 0 0 0 10 10 10 -97 97 97 -98 98 98 100 100 100 103 103 103 +97 97 97 +98 98 98 PREHOOK: query: drop table skew_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@skew_mm @@ -966,15 +967,15 @@ POSTHOOK: Input: default@skew_dp_union_mm@k3=98 0 0 0 0 1 2 3 4 10 10 10 10 +100 100 100 100 +101 102 103 104 +103 103 103 103 +104 105 106 107 11 12 13 14 97 97 97 97 98 98 98 98 98 99 100 101 99 100 101 102 -100 100 100 100 -101 102 103 104 -103 103 103 103 -104 105 106 107 PREHOOK: query: drop table skew_dp_union_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@skew_dp_union_mm @@ -1014,12 +1015,12 @@ POSTHOOK: query: select * from merge0_mm POSTHOOK: type: QUERY POSTHOOK: Input: default@merge0_mm #### A masked pattern was here #### -98 -97 0 10 100 103 +97 +98 PREHOOK: query: insert into table merge0_mm select key from intermediate_n0 PREHOOK: type: QUERY PREHOOK: Input: default@intermediate_n0 @@ -1043,18 +1044,18 @@ POSTHOOK: query: select * from merge0_mm POSTHOOK: type: QUERY POSTHOOK: Input: default@merge0_mm #### A masked pattern was here #### -98 -97 0 -10 -100 -103 -98 -97 0 10 +10 +100 100 103 +103 +97 +97 +98 +98 PREHOOK: query: drop table merge0_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@merge0_mm @@ -1094,12 +1095,12 @@ POSTHOOK: query: select * from merge2_mm POSTHOOK: type: QUERY POSTHOOK: Input: default@merge2_mm #### A masked pattern was here #### -98 -97 0 10 100 103 +97 +98 PREHOOK: query: insert into table merge2_mm select key from intermediate_n0 PREHOOK: type: QUERY PREHOOK: Input: default@intermediate_n0 @@ -1123,18 +1124,18 @@ POSTHOOK: query: select * from merge2_mm POSTHOOK: type: QUERY POSTHOOK: Input: default@merge2_mm #### A masked pattern was here #### -98 -97 0 -10 -100 -103 -98 -97 0 10 +10 100 +100 +103 103 +97 +97 +98 +98 PREHOOK: query: drop table merge2_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@merge2_mm @@ -1198,10 +1199,10 @@ POSTHOOK: Input: default@merge1_mm@key=98 #### A masked pattern was here #### 0 0 10 10 -97 97 -98 98 100 100 103 103 +97 97 +98 98 PREHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate_n0 PREHOOK: type: QUERY PREHOOK: Input: default@intermediate_n0 @@ -1251,14 +1252,14 @@ POSTHOOK: Input: default@merge1_mm@key=98 0 0 10 10 10 10 -97 97 -97 97 -98 98 -98 98 100 100 100 100 103 103 103 103 +97 97 +97 97 +98 98 +98 98 PREHOOK: query: drop table merge1_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@merge1_mm @@ -1297,12 +1298,12 @@ POSTHOOK: query: select * from ctas0_mm POSTHOOK: type: QUERY POSTHOOK: Input: default@ctas0_mm #### A masked pattern was here #### -98 455 -97 455 0 456 10 456 100 457 103 457 +97 455 +98 455 PREHOOK: query: drop table ctas0_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@ctas0_mm @@ -1343,10 +1344,6 @@ POSTHOOK: query: select * from ctas1_mm POSTHOOK: type: QUERY POSTHOOK: Input: default@ctas1_mm #### A masked pattern was here #### -98 455 -98 455 -97 455 -97 455 0 456 0 456 10 456 @@ -1355,6 +1352,10 @@ POSTHOOK: Input: default@ctas1_mm 100 457 103 457 103 457 +97 455 +97 455 +98 455 +98 455 PREHOOK: query: drop table ctas1_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@ctas1_mm @@ -1421,10 +1422,10 @@ POSTHOOK: Input: default@multi0_1_mm #### A masked pattern was here #### 0 456 10 456 -97 455 -98 455 100 457 103 457 +97 455 +98 455 PREHOOK: query: select * from multi0_2_mm order by key, key2 PREHOOK: type: QUERY PREHOOK: Input: default@multi0_2_mm @@ -1473,8 +1474,6 @@ POSTHOOK: Input: default@multi0_1_mm #### A masked pattern was here #### 0 456 10 456 -97 455 -98 455 100 457 103 457 455 97 @@ -1483,6 +1482,8 @@ POSTHOOK: Input: default@multi0_1_mm 456 10 457 100 457 103 +97 455 +98 455 PREHOOK: query: select * from multi0_2_mm order by key, key2 PREHOOK: type: QUERY PREHOOK: Input: default@multi0_2_mm @@ -1493,10 +1494,10 @@ POSTHOOK: Input: default@multi0_2_mm #### A masked pattern was here #### 0 456 10 456 -97 455 -98 455 100 457 103 457 +97 455 +98 455 PREHOOK: query: drop table multi0_1_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@multi0_1_mm @@ -1563,8 +1564,6 @@ POSTHOOK: Input: default@multi1_mm@p=2 #### A masked pattern was here #### 0 456 2 10 456 2 -97 455 2 -98 455 2 100 457 2 103 457 2 455 97 1 @@ -1573,6 +1572,8 @@ POSTHOOK: Input: default@multi1_mm@p=2 456 10 1 457 100 1 457 103 1 +97 455 2 +98 455 2 PREHOOK: query: from intermediate_n0 insert into table multi1_mm partition(p=2) select p, key insert overwrite table multi1_mm partition(p=1) select key, p @@ -1613,10 +1614,6 @@ POSTHOOK: Input: default@multi1_mm@p=2 0 456 2 10 456 1 10 456 2 -97 455 1 -97 455 2 -98 455 1 -98 455 2 100 457 1 100 457 2 103 457 1 @@ -1627,6 +1624,10 @@ POSTHOOK: Input: default@multi1_mm@p=2 456 10 2 457 100 2 457 103 2 +97 455 1 +97 455 2 +98 455 1 +98 455 2 PREHOOK: query: from intermediate_n0 insert into table multi1_mm partition(p) select p, key, p insert into table multi1_mm partition(p=1) select key, p @@ -1683,12 +1684,6 @@ POSTHOOK: Input: default@multi1_mm@p=457 10 456 1 10 456 1 10 456 2 -97 455 1 -97 455 1 -97 455 2 -98 455 1 -98 455 1 -98 455 2 100 457 1 100 457 1 100 457 2 @@ -1707,6 +1702,12 @@ POSTHOOK: Input: default@multi1_mm@p=457 457 100 457 457 103 2 457 103 457 +97 455 1 +97 455 1 +97 455 2 +98 455 1 +98 455 1 +98 455 2 PREHOOK: query: from intermediate_n0 insert into table multi1_mm partition(p) select p, key, 1 insert into table multi1_mm partition(p=1) select key, p @@ -1756,14 +1757,6 @@ POSTHOOK: Input: default@multi1_mm@p=457 10 456 1 10 456 1 10 456 2 -97 455 1 -97 455 1 -97 455 1 -97 455 2 -98 455 1 -98 455 1 -98 455 1 -98 455 2 100 457 1 100 457 1 100 457 1 @@ -1790,6 +1783,14 @@ POSTHOOK: Input: default@multi1_mm@p=457 457 103 1 457 103 2 457 103 457 +97 455 1 +97 455 1 +97 455 1 +97 455 2 +98 455 1 +98 455 1 +98 455 1 +98 455 2 PREHOOK: query: drop table multi1_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@multi1_mm @@ -1843,7 +1844,7 @@ Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} bucketing_version 2 - numFiles 1 + numFiles 3 numRows 6 rawDataSize 13 totalSize 19 @@ -1894,7 +1895,7 @@ Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} bucketing_version 2 - numFiles 2 + numFiles 6 numRows 12 rawDataSize 26 totalSize 38 @@ -2136,12 +2137,12 @@ POSTHOOK: query: SELECT * FROM temp1 POSTHOOK: type: QUERY POSTHOOK: Input: default@temp1 #### A masked pattern was here #### -98 -97 0 10 100 103 +97 +98 PREHOOK: query: drop table intermediate_n0 PREHOOK: type: DROPTABLE PREHOOK: Input: default@intermediate_n0