From f4099f60113a7adaabb55ab00cbfd15e4626d164 Mon Sep 17 00:00:00 2001 From: Naresh P R Date: Mon, 11 Nov 2019 20:05:29 +0530 Subject: [PATCH] HIVE-22478 - Import command fails from lower version to higher version when hive.strict.managed.tables enabled --- .../hive/ql/parse/ImportSemanticAnalyzer.java | 7 +- .../clientpositive/import_non_acid_to_acid.q | 14 ++ .../import_non_acid_to_acid.q.out | 132 ++++++++++++++++++ 3 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/import_non_acid_to_acid.q create mode 100644 ql/src/test/results/clientpositive/import_non_acid_to_acid.q.out diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index 85c5360699..45fa085340 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -283,17 +283,18 @@ public static boolean prepareImport(boolean isImportCmd, org.apache.hadoop.hive.metastore.api.Table tblObj = rv.getTable(); try { // The table can be non acid in case of replication from a cluster with STRICT_MANAGED set to false. - if (!TxnUtils.isTransactionalTable(tblObj) && replicationSpec.isInReplicationScope() && + if (!TxnUtils.isTransactionalTable(tblObj) && x.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STRICT_MANAGED_TABLES) && (TableType.valueOf(tblObj.getTableType()) == TableType.MANAGED_TABLE)) { //TODO : dump metadata should be read to make sure that migration is required. upgradeTableDesc(tblObj, rv, x); //if the conversion is from non transactional to transactional table - if (TxnUtils.isTransactionalTable(tblObj)) { + if (TxnUtils.isTransactionalTable(tblObj) && replicationSpec.isInReplicationScope()) { replicationSpec.setMigratingToTxnTable(); } tblDesc = getBaseCreateTableDescFromTable(dbname, tblObj); - if (TableType.valueOf(tblObj.getTableType()) == TableType.EXTERNAL_TABLE) { + if (TableType.valueOf(tblObj.getTableType()) == TableType.EXTERNAL_TABLE && + replicationSpec.isInReplicationScope()) { replicationSpec.setMigratingToExternalTable(); tblDesc.setExternal(true); // we should set this to null so default location for external tables is chosen on target diff --git a/ql/src/test/queries/clientpositive/import_non_acid_to_acid.q b/ql/src/test/queries/clientpositive/import_non_acid_to_acid.q new file mode 100644 index 0000000000..57ca5e8210 --- /dev/null +++ b/ql/src/test/queries/clientpositive/import_non_acid_to_acid.q @@ -0,0 +1,14 @@ +create table test_table(age int, name string) stored as orc; +insert into test_table values(1, "abcd"); +describe formatted test_table; +select * from test_table; +export table test_table to 'ql/test/data/exports/test_table'; +drop table test_table; +-- Test import from non-acid managed table to acid managed table +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.managed.tables=true; +set metastore.create.as.acid=true; +import table test_table from 'ql/test/data/exports/test_table'; +describe formatted test_table; +select * from test_table; \ No newline at end of file diff --git a/ql/src/test/results/clientpositive/import_non_acid_to_acid.q.out b/ql/src/test/results/clientpositive/import_non_acid_to_acid.q.out new file mode 100644 index 0000000000..2d594f2e4e --- /dev/null +++ b/ql/src/test/results/clientpositive/import_non_acid_to_acid.q.out @@ -0,0 +1,132 @@ +PREHOOK: query: create table test_table(age int, name string) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_table +POSTHOOK: query: create table test_table(age int, name string) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_table +PREHOOK: query: insert into test_table values(1, "abcd") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test_table +POSTHOOK: query: insert into test_table values(1, "abcd") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test_table +POSTHOOK: Lineage: test_table.age SCRIPT [] +POSTHOOK: Lineage: test_table.name SCRIPT [] +PREHOOK: query: describe formatted test_table +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@test_table +POSTHOOK: query: describe formatted test_table +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@test_table +# col_name data_type comment +age int +name string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"age\":\"true\",\"name\":\"true\"}} + bucketing_version 2 + numFiles 1 + numRows 1 + rawDataSize 92 + totalSize 301 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from test_table +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table +#### A masked pattern was here #### +POSTHOOK: query: select * from test_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table +#### A masked pattern was here #### +1 abcd +PREHOOK: query: export table test_table to 'ql/test/data/exports/test_table' +PREHOOK: type: EXPORT +PREHOOK: Input: default@test_table +#### A masked pattern was here #### +POSTHOOK: query: export table test_table to 'ql/test/data/exports/test_table' +POSTHOOK: type: EXPORT +POSTHOOK: Input: default@test_table +#### A masked pattern was here #### +PREHOOK: query: drop table test_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@test_table +PREHOOK: Output: default@test_table +POSTHOOK: query: drop table test_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@test_table +POSTHOOK: Output: default@test_table +PREHOOK: query: import table test_table from 'ql/test/data/exports/test_table' +PREHOOK: type: IMPORT +#### A masked pattern was here #### +PREHOOK: Output: database:default +POSTHOOK: query: import table test_table from 'ql/test/data/exports/test_table' +POSTHOOK: type: IMPORT +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_table +PREHOOK: query: describe formatted test_table +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@test_table +POSTHOOK: query: describe formatted test_table +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@test_table +# col_name data_type comment +age int +name string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + bucketing_version 2 + numFiles 1 + numRows 0 + rawDataSize 0 + totalSize 301 + transactional true + transactional_properties default +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from test_table +PREHOOK: type: QUERY +PREHOOK: Input: default@test_table +#### A masked pattern was here #### +POSTHOOK: query: select * from test_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_table +#### A masked pattern was here #### +1 abcd -- 2.18.0