diff --git ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index 183515a0ed..4b9a8a1ee6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -1932,4 +1932,9 @@ public static Long extractWriteId(Path file) { } return writeId; } + + public static void setNonTransactional(Map tblProps) { + tblProps.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "false"); + tblProps.remove(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES); + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index 682b641b21..cc7f0d5ca0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -255,16 +255,13 @@ public static boolean prepareImport(boolean isImportCmd, } if (isExternalSet) { - if (AcidUtils.isInsertOnlyTable(tblDesc.getTblProps())) { - throw new SemanticException("Cannot import an MM table as external"); - } tblDesc.setExternal(isExternalSet); // This condition-check could have been avoided, but to honour the old // default of not calling if it wasn't set, we retain that behaviour. // TODO:cleanup after verification that the outer if isn't really needed here } - if (isLocationSet){ + if (isLocationSet) { tblDesc.setLocation(parsedLocation); x.getInputs().add(toReadEntity(new Path(parsedLocation), x.getConf())); } @@ -320,11 +317,16 @@ public static boolean prepareImport(boolean isImportCmd, boolean tableExists = false; if (table != null) { - checkTable(table, tblDesc,replicationSpec, x.getConf()); + checkTable(table, tblDesc, replicationSpec, x.getConf()); x.getLOG().debug("table " + tblDesc.getTableName() + " exists: metadata checked"); tableExists = true; } + if (!tableExists && isExternalSet) { + // If the user is explicitly importing a new external table, clear txn flags from the spec. + AcidUtils.setNonTransactional(tblDesc.getTblProps()); + } + Long writeId = 0L; // Initialize with 0 for non-ACID and non-MM tables. int stmtId = 0; if (!replicationSpec.isInReplicationScope() @@ -855,7 +857,7 @@ private static void createRegularImportTasks( x.getLOG().debug("table " + tblDesc.getTableName() + " does not exist"); Task t = createTableTask(tblDesc, x); - table = createNewTableMetadataObject(tblDesc); + table = createNewTableMetadataObject(tblDesc, false); Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName()); @@ -891,14 +893,19 @@ private static void createRegularImportTasks( } } - private static Table createNewTableMetadataObject(ImportTableDesc tblDesc) + private static Table createNewTableMetadataObject(ImportTableDesc tblDesc, boolean isRepl) throws SemanticException { Table newTable = new Table(tblDesc.getDatabaseName(), tblDesc.getTableName()); //so that we know the type of table we are creating: acid/MM to match what was exported newTable.setParameters(tblDesc.getTblProps()); if(tblDesc.isExternal() && AcidUtils.isTransactionalTable(newTable)) { - throw new SemanticException("External tables may not be transactional: " + - Warehouse.getQualifiedName(tblDesc.getDatabaseName(), tblDesc.getTableName())); + if (isRepl) { + throw new SemanticException("External tables may not be transactional: " + + Warehouse.getQualifiedName(tblDesc.getDatabaseName(), tblDesc.getTableName())); + } else { + throw new AssertionError("Internal error: transactional properties not set properly" + + tblDesc.getTblProps()); + } } return newTable; } @@ -992,7 +999,7 @@ private static void createReplImportTasks( } Task t = createTableTask(tblDesc, x); - table = createNewTableMetadataObject(tblDesc); + table = createNewTableMetadataObject(tblDesc, true); if (!replicationSpec.isMetadataOnly()) { if (isPartitioned(tblDesc)) { @@ -1025,7 +1032,7 @@ private static void createReplImportTasks( // create the dummy table object for adding repl tasks. boolean isOldTableValid = true; if (table.isPartitioned() != isPartitioned(tblDesc)) { - table = createNewTableMetadataObject(tblDesc); + table = createNewTableMetadataObject(tblDesc, true); isOldTableValid = false; } @@ -1043,7 +1050,7 @@ private static void createReplImportTasks( ptn = x.getHive().getPartition(table, partSpec, false); } catch (HiveException ex) { ptn = null; - table = createNewTableMetadataObject(tblDesc); + table = createNewTableMetadataObject(tblDesc, true); isOldTableValid = false; } } diff --git ql/src/test/queries/clientpositive/mm_exim.q ql/src/test/queries/clientpositive/mm_exim.q index 9870bf4ed8..13b3055d96 100644 --- ql/src/test/queries/clientpositive/mm_exim.q +++ ql/src/test/queries/clientpositive/mm_exim.q @@ -97,4 +97,12 @@ import table import7_mm from 'ql/test/data/exports/intermmediate_part'; select * from import7_mm order by key, p; drop table import7_mm; +-- import MM as external + +drop table import8_mm; +import external table import8_mm from 'ql/test/data/exports/intermmediate_nonpart'; +desc formatted import8_mm; +select * from import8_mm order by key, p; +drop table import8_mm; + set hive.exim.test.mode=false; \ No newline at end of file diff --git ql/src/test/results/clientpositive/llap/mm_exim.q.out ql/src/test/results/clientpositive/llap/mm_exim.q.out index 001cd055c2..8dfc738fb7 100644 --- ql/src/test/results/clientpositive/llap/mm_exim.q.out +++ ql/src/test/results/clientpositive/llap/mm_exim.q.out @@ -610,3 +610,73 @@ POSTHOOK: query: drop table import7_mm POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@import7_mm POSTHOOK: Output: default@import7_mm +PREHOOK: query: drop table import8_mm +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table import8_mm +POSTHOOK: type: DROPTABLE +PREHOOK: query: import external table import8_mm from 'ql/test/data/exports/intermmediate_nonpart' +PREHOOK: type: IMPORT +#### A masked pattern was here #### +PREHOOK: Output: database:default +POSTHOOK: query: import external table import8_mm from 'ql/test/data/exports/intermmediate_nonpart' +POSTHOOK: type: IMPORT +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@import8_mm +PREHOOK: query: desc formatted import8_mm +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@import8_mm +POSTHOOK: query: desc formatted import8_mm +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@import8_mm +# col_name data_type comment +key int +p int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + EXTERNAL TRUE + bucketing_version 2 + numFiles 3 + numRows 6 + rawDataSize 37 + totalSize 43 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from import8_mm order by key, p +PREHOOK: type: QUERY +PREHOOK: Input: default@import8_mm +#### A masked pattern was here #### +POSTHOOK: query: select * from import8_mm order by key, p +POSTHOOK: type: QUERY +POSTHOOK: Input: default@import8_mm +#### A masked pattern was here #### +0 456 +10 456 +97 455 +98 455 +100 457 +103 457 +PREHOOK: query: drop table import8_mm +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@import8_mm +PREHOOK: Output: default@import8_mm +POSTHOOK: query: drop table import8_mm +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@import8_mm +POSTHOOK: Output: default@import8_mm