commit b7218840d10acfe63c7637f82ac7b004ecb44f6d Author: Ivan Suller Date: Fri Apr 12 11:02:33 2019 +0200 HIVE-21714 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index cd13397a79..9ad4e71482 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -98,7 +98,7 @@ **/ @SuppressWarnings("deprecation") public class FileSinkOperator extends TerminalOperator implements - Serializable { + Serializable, IConfigureJobConf { public static final Logger LOG = LoggerFactory.getLogger(FileSinkOperator.class); @@ -1260,13 +1260,12 @@ public void closeOp(boolean abort) throws HiveException { } if (!bDynParts && !filesCreated) { - boolean skipFiles = "tez".equalsIgnoreCase( + boolean isTez = "tez".equalsIgnoreCase( HiveConf.getVar(hconf, ConfVars.HIVE_EXECUTION_ENGINE)); - if (skipFiles) { - Class clazz = conf.getTableInfo().getOutputFileFormatClass(); - skipFiles = !StreamingOutputFormat.class.isAssignableFrom(clazz); - } - if (!skipFiles) { + Class clazz = conf.getTableInfo().getOutputFileFormatClass(); + boolean isStreaming = StreamingOutputFormat.class.isAssignableFrom(clazz); + + if (!isTez || isStreaming || this.isInsertOverwrite) { createBucketFiles(fsp); } } @@ -1607,4 +1606,10 @@ private boolean isNativeTable() { return !conf.getTableInfo().isNonNative(); } + @Override + public void configureJobConf(JobConf job) { + if (conf.getInsertOverwrite()) { + job.setBoolean(Utilities.ENSURE_OPERATORS_EXECUTED, true); + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 55ae535df5..287f979d1e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -6069,3 +6069,4 @@ public StorageHandlerInfo getStorageHandlerInfo(Table table) } } } + diff --git a/ql/src/test/queries/clientpositive/insert_overwrite.q b/ql/src/test/queries/clientpositive/insert_overwrite.q index 12dd1b57a2..f45c79de2e 100644 --- a/ql/src/test/queries/clientpositive/insert_overwrite.q +++ b/ql/src/test/queries/clientpositive/insert_overwrite.q @@ -3,9 +3,15 @@ set hive.stats.column.autogather=false; set hive.stats.autogather=false; set hive.compute.query.using.stats=false; +set hive.create.as.insert.only=true; +set hive.default.fileformat.managed=ORC; +set hive.strict.managed.tables=true; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + CREATE EXTERNAL TABLE ext_non_part (col string); INSERT INTO ext_non_part VALUES ('first'), ('second'); -CREATE TABLE b LIKE ext_non_part; +CREATE TABLE b (col string); INSERT OVERWRITE TABLE ext_non_part SELECT * FROM b; diff --git a/ql/src/test/results/clientpositive/llap/insert_overwrite.q.out b/ql/src/test/results/clientpositive/llap/insert_overwrite.q.out index 68f7cc895d..39e12958c6 100644 --- a/ql/src/test/results/clientpositive/llap/insert_overwrite.q.out +++ b/ql/src/test/results/clientpositive/llap/insert_overwrite.q.out @@ -15,11 +15,11 @@ POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ext_non_part POSTHOOK: Lineage: ext_non_part.col SCRIPT [] -PREHOOK: query: CREATE TABLE b LIKE ext_non_part +PREHOOK: query: CREATE TABLE b (col string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@b -POSTHOOK: query: CREATE TABLE b LIKE ext_non_part +POSTHOOK: query: CREATE TABLE b (col string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@b