diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java index 3427b59e67..7ef8d79c73 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java @@ -20,12 +20,15 @@ import com.google.common.collect.Collections2; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.common.repl.ReplScope; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.ddl.DDLWork; import org.apache.hadoop.hive.ql.ddl.database.alter.poperties.AlterDatabaseSetPropertiesDesc; +import org.apache.hadoop.hive.ql.ddl.view.create.CreateViewDesc; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.BootstrapEvent; @@ -33,9 +36,9 @@ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.DatabaseEvent; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.FunctionEvent; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.PartitionEvent; -import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.TableEvent; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.filesystem.BootstrapEventsIterator; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.filesystem.ConstraintEventsIterator; +import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.filesystem.FSTableEvent; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.LoadConstraint; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.LoadDatabase; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.LoadFunction; @@ -50,10 +53,13 @@ import org.apache.hadoop.hive.ql.exec.util.DAGTraversal; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.HiveTableName; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.repl.ReplLogger; +import org.apache.hadoop.hive.ql.parse.repl.load.MetaData; import org.apache.hadoop.hive.ql.plan.api.StageType; import java.io.Serializable; @@ -67,6 +73,7 @@ import static org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.LoadDatabase.AlterDatabase; public class ReplLoadTask extends Task implements Serializable { + private static final long serialVersionUID = 1L; private final static int ZERO_TASKS = 0; @Override @@ -84,7 +91,7 @@ public StageType getType() { * by the driver. It does not track details across multiple runs of LoadTask. */ private static class Scope { - boolean database = false, table = false, partition = false; + boolean database = false, table = false; List> rootTasks = new ArrayList<>(); } @@ -163,10 +170,34 @@ a database ( directory ) the dbTracker / tableTracker are setup correctly always. */ TableContext tableContext = new TableContext(dbTracker, work.dbNameToLoadIn); - TableEvent tableEvent = (TableEvent) next; - LoadTable loadTable = new LoadTable(tableEvent, loadContext, iterator.replLogger(), - tableContext, loadTaskTracker); - tableTracker = loadTable.tasks(work.isIncrementalLoad()); + FSTableEvent tableEvent = (FSTableEvent) next; + if (TableType.VIRTUAL_VIEW.name().equals(tableEvent.getMetaData().getTable().getTableType())) { + tableTracker = new TaskTracker(1); + tableTracker.addTask(createViewTask(tableEvent.getMetaData(), work.dbNameToLoadIn, conf)); + } else { + LoadTable loadTable = new LoadTable(tableEvent, loadContext, iterator.replLogger(), tableContext, + loadTaskTracker); + tableTracker = loadTable.tasks(work.isIncrementalLoad()); + + /* + for table replication if we reach the max number of tasks then for the next run we will + try to reload the same table again, this is mainly for ease of understanding the code + as then we can avoid handling == > loading partitions for the table given that + the creation of table lead to reaching max tasks vs, loading next table since current + one does not have partitions. + */ + + // for a table we explicitly try to load partitions as there is no separate partitions events. + LoadPartitions loadPartitions = + new LoadPartitions(loadContext, iterator.replLogger(), loadTaskTracker, tableEvent, + work.dbNameToLoadIn, tableContext); + TaskTracker partitionsTracker = loadPartitions.tasks(); + partitionsPostProcessing(iterator, scope, loadTaskTracker, tableTracker, + partitionsTracker); + tableTracker.debugLog("table"); + partitionsTracker.debugLog("partitions for table"); + } + setUpDependencies(dbTracker, tableTracker); if (!scope.database && tableTracker.hasTasks()) { scope.rootTasks.addAll(tableTracker.tasks()); @@ -176,24 +207,6 @@ a database ( directory ) // tracker has no tasks. scope.table = false; } - - /* - for table replication if we reach the max number of tasks then for the next run we will - try to reload the same table again, this is mainly for ease of understanding the code - as then we can avoid handling == > loading partitions for the table given that - the creation of table lead to reaching max tasks vs, loading next table since current - one does not have partitions. - */ - - // for a table we explicitly try to load partitions as there is no separate partitions events. - LoadPartitions loadPartitions = - new LoadPartitions(loadContext, iterator.replLogger(), loadTaskTracker, tableEvent, - work.dbNameToLoadIn, tableContext); - TaskTracker partitionsTracker = loadPartitions.tasks(); - partitionsPostProcessing(iterator, scope, loadTaskTracker, tableTracker, - partitionsTracker); - tableTracker.debugLog("table"); - partitionsTracker.debugLog("partitions for table"); break; } case Partition: { @@ -282,6 +295,32 @@ a database ( directory ) return 0; } + public static Task createViewTask(MetaData metaData, String dbNameToLoadIn, HiveConf conf) + throws SemanticException { + Table table = new Table(metaData.getTable()); + String dbName = dbNameToLoadIn == null ? table.getDbName() : dbNameToLoadIn; + TableName tableName = HiveTableName.ofNullable(table.getTableName(), dbName); + String dbDotView = tableName.getNotEmptyDbTable(); + CreateViewDesc desc = new CreateViewDesc(dbDotView, table.getAllCols(), null, table.getParameters(), + table.getPartColNames(), false, false, false, table.getSd().getInputFormat(), table.getSd().getOutputFormat(), + table.getSd().getSerdeInfo().getSerializationLib()); + String originalText = table.getViewOriginalText(); + String expandedText = table.getViewExpandedText(); + + if (!dbName.equals(table.getDbName())) { + // TODO: If the DB name doesn't match with the metadata from dump, then need to rewrite the original and expanded + // texts using new DB name. Currently it refers to the source database name. + } + + desc.setViewOriginalText(originalText); + desc.setViewExpandedText(expandedText); + desc.setPartCols(table.getPartCols()); + desc.setReplicationSpec(metaData.getReplicationSpec()); + desc.setOwnerName(table.getOwner()); + + return TaskFactory.get(new DDLWork(new HashSet<>(), new HashSet<>(), desc), conf); + } + /** * If replication policy is changed between previous and current load, then the excluded tables in * the new replication policy will be dropped. @@ -380,7 +419,6 @@ private void partitionsPostProcessing(BootstrapEventsIterator iterator, setUpDependencies(tableTracker, partitionsTracker); if (!scope.database && !scope.table) { scope.rootTasks.addAll(partitionsTracker.tasks()); - scope.partition = true; } loadTaskTracker.update(tableTracker); loadTaskTracker.update(partitionsTracker); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/FSTableEvent.java ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/FSTableEvent.java index 6d38c0309f..d2f02143ce 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/FSTableEvent.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/FSTableEvent.java @@ -85,6 +85,10 @@ public Path metadataPath() { return fromPath; } + public MetaData getMetaData() { + return metadata; + } + /** * To determine if the tableDesc is for an external table, * use {@link ImportTableDesc#isExternal()} diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java index 8ab39433ec..093847e9ef 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java @@ -121,6 +121,10 @@ private void analyzeAcidExport(ASTNode ast) throws SemanticException { assert tableTree != null && tableTree.getType() == HiveParser.TOK_TAB; ASTNode tokRefOrNameExportTable = (ASTNode) tableTree.getChild(0); Table exportTable = getTargetTable(tokRefOrNameExportTable); + + if (exportTable != null && (exportTable.isView() || exportTable.isMaterializedView())) { + throw new SemanticException("Views and Materialized Views can not be exported."); + } assert AcidUtils.isFullAcidTable(exportTable); //need to create the table "manually" rather than creating a task since it has to exist to diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java index bc90ea1db7..815f275092 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hive.ql.parse.repl.dump.io.TableSerializer; import org.apache.hadoop.hive.ql.parse.repl.load.MetaData; import org.apache.hadoop.hive.ql.parse.repl.load.MetadataJson; +import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.thrift.TException; import org.json.JSONException; import org.slf4j.Logger; @@ -365,6 +366,19 @@ public static void createExportDump(FileSystem fs, Path metadataPath, Table tabl } } + public static MetaData getMetaDataFromLocation(String fromLocn, HiveConf conf) + throws SemanticException, IOException { + URI fromURI = getValidatedURI(conf, PlanUtils.stripQuotes(fromLocn)); + Path fromPath = new Path(fromURI.getScheme(), fromURI.getAuthority(), fromURI.getPath()); + FileSystem fs = FileSystem.get(fromURI, conf); + + try { + return readMetaData(fs, new Path(fromPath, EximUtil.METADATA_NAME)); + } catch (IOException e) { + throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e); + } + } + public static MetaData readMetaData(FileSystem fs, Path metadataPath) throws IOException, SemanticException { String message = readAsString(fs, metadataPath); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java index 8c82379343..debb01296d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java @@ -100,6 +100,10 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { } } + if (ts != null && (ts.tableHandle.isView() || ts.tableHandle.isMaterializedView())) { + throw new SemanticException("Views and Materialized Views can not be exported."); + } + // initialize export path String tmpPath = stripQuotes(toTree.getText()); // All parsing is done, we're now good to start the export process diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index 7354a3e7e0..ba1abd341d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -78,7 +78,6 @@ import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils; import java.io.IOException; -import java.io.Serializable; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -168,12 +167,13 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { parsedDbName = SessionState.get().getCurrentDatabase(); } // parsing statement is now done, on to logic. + EximUtil.SemanticAnalyzerWrapperContext x = + new EximUtil.SemanticAnalyzerWrapperContext(conf, db, inputs, outputs, rootTasks, LOG, ctx); + MetaData rv = EximUtil.getMetaDataFromLocation(fromTree.getText(), x.getConf()); tableExists = prepareImport(true, isLocationSet, isExternalSet, isPartSpecSet, waitOnPrecursor, parsedLocation, parsedTableName, parsedDbName, parsedPartSpec, fromTree.getText(), - new EximUtil.SemanticAnalyzerWrapperContext(conf, db, inputs, outputs, rootTasks, LOG, ctx), - - null, getTxnMgr(), 0); + x, null, getTxnMgr(), 0, rv); } catch (SemanticException e) { throw e; @@ -237,7 +237,8 @@ public static boolean prepareImport(boolean isImportCmd, LinkedHashMap parsedPartSpec, String fromLocn, EximUtil.SemanticAnalyzerWrapperContext x, UpdatedMetaDataTracker updatedMetadata, HiveTxnManager txnMgr, - long writeId // Initialize with 0 for non-ACID and non-MM tables. + long writeId, // Initialize with 0 for non-ACID and non-MM tables. + MetaData rv ) throws IOException, MetaException, HiveException, URISyntaxException { // initialize load path @@ -247,13 +248,6 @@ public static boolean prepareImport(boolean isImportCmd, FileSystem fs = FileSystem.get(fromURI, x.getConf()); x.getInputs().add(toReadEntity(fromPath, x.getConf())); - MetaData rv; - try { - rv = EximUtil.readMetaData(fs, new Path(fromPath, EximUtil.METADATA_NAME)); - } catch (IOException e) { - throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e); - } - if (rv.getTable() == null) { // nothing to do here, silently return. return false; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java index f6d57e86f5..fda6c6c7fe 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java @@ -17,14 +17,13 @@ */ package org.apache.hadoop.hive.ql.parse.repl.load.message; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage; import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage; -import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.repl.ReplExternalTables; +import org.apache.hadoop.hive.ql.exec.repl.ReplLoadTask; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer; @@ -34,9 +33,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.Serializable; -import java.net.URI; import java.util.ArrayList; import java.util.List; @@ -53,20 +49,13 @@ DumpType eventType = context.dmd.getDumpType(); Tuple tuple = extract(context); + MetaData rv = EximUtil.getMetaDataFromLocation(context.location, context.hiveConf); + if (tuple.isExternalTable) { - URI fromURI = EximUtil.getValidatedURI(context.hiveConf, context.location); - Path fromPath = new Path(fromURI.getScheme(), fromURI.getAuthority(), fromURI.getPath()); isLocationSet = true; isExternal = true; - FileSystem fs = FileSystem.get(fromURI, context.hiveConf); - try { - MetaData rv = EximUtil.readMetaData(fs, new Path(fromPath, EximUtil.METADATA_NAME)); - Table table = new Table(rv.getTable()); - parsedLocation = ReplExternalTables - .externalTableLocation(context.hiveConf, table.getSd().getLocation()); - } catch (IOException e) { - throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e); - } + Table table = new Table(rv.getTable()); + parsedLocation = ReplExternalTables.externalTableLocation(context.hiveConf, table.getSd().getLocation()); } context.nestedContext.setConf(context.hiveConf); @@ -77,9 +66,13 @@ x.setEventType(eventType); // REPL LOAD is not partition level. It is always DB or table level. So, passing null for partition specs. - ImportSemanticAnalyzer.prepareImport(false, isLocationSet, isExternal, false, - (context.precursor != null), parsedLocation, null, context.dbName, - null, context.location, x, updatedMetadata, context.getTxnMgr(), tuple.writeId); + if (TableType.VIRTUAL_VIEW.name().equals(rv.getTable().getTableType())) { + importTasks.add(ReplLoadTask.createViewTask(rv, context.dbName, context.hiveConf)); + } else { + ImportSemanticAnalyzer.prepareImport(false, isLocationSet, isExternal, false, + (context.precursor != null), parsedLocation, null, context.dbName, + null, context.location, x, updatedMetadata, context.getTxnMgr(), tuple.writeId, rv); + } Task openTxnTask = x.getOpenTxnTask(); if (openTxnTask != null && !importTasks.isEmpty()) { diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java index dc7040e388..e74f53a431 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java @@ -22,7 +22,6 @@ import java.util.Map; import java.util.Set; -import com.google.common.collect.ImmutableSet; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; @@ -30,12 +29,10 @@ import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.ql.ddl.DDLWork; import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableDesc; -import org.apache.hadoop.hive.ql.ddl.view.create.CreateViewDesc; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.HiveTableName; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; @@ -47,343 +44,138 @@ */ public class ImportTableDesc { private String dbName = null; - private Table table = null; private CreateTableDesc createTblDesc = null; - private CreateViewDesc createViewDesc = null; - - public enum TYPE { TABLE, VIEW }; public ImportTableDesc(String dbName, Table table) throws Exception { this.dbName = dbName; - this.table = table; - final TableName tableName = HiveTableName.ofNullable(table.getTableName(), dbName); - - switch (getDescType()) { - case TABLE: - this.createTblDesc = new CreateTableDesc(tableName, - false, // isExternal: set to false here, can be overwritten by the IMPORT stmt - false, - table.getSd().getCols(), - table.getPartitionKeys(), - table.getSd().getBucketCols(), - table.getSd().getSortCols(), - table.getSd().getNumBuckets(), - null, null, null, null, null, // these 5 delims passed as serde params - null, // comment passed as table params - table.getSd().getInputFormat(), - table.getSd().getOutputFormat(), - null, // location: set to null here, can be overwritten by the IMPORT stmt - table.getSd().getSerdeInfo().getSerializationLib(), - null, // storagehandler passed as table params - table.getSd().getSerdeInfo().getParameters(), - table.getParameters(), false, - (null == table.getSd().getSkewedInfo()) ? null : table.getSd().getSkewedInfo() - .getSkewedColNames(), - (null == table.getSd().getSkewedInfo()) ? null : table.getSd().getSkewedInfo() - .getSkewedColValues(), - null, - null, - null, - null, - null, - null, - table.getColStats(), - table.getTTable().getWriteId()); - this.createTblDesc.setStoredAsSubDirectories(table.getSd().isStoredAsSubDirectories()); - break; - case VIEW: - final String dbDotView = tableName.getNotEmptyDbTable(); - if (table.isMaterializedView()) { - this.createViewDesc = new CreateViewDesc(dbDotView, - table.getAllCols(), - null, // comment passed as table params - table.getParameters(), - table.getPartColNames(), - null, // sort columns passed as table params (if present) - null, // distribute columns passed as table params (if present) - false,false,false,false, - table.getSd().getInputFormat(), - table.getSd().getOutputFormat(), - null, // location: set to null here, can be overwritten by the IMPORT stmt - table.getSd().getSerdeInfo().getSerializationLib(), - null, // storagehandler passed as table params - table.getSd().getSerdeInfo().getParameters()); - // TODO: If the DB name from the creation metadata for any of the tables has changed, - // we should update it. Currently it refers to the source database name. - this.createViewDesc.setTablesUsed(table.getCreationMetadata() != null ? - table.getCreationMetadata().getTablesUsed() : ImmutableSet.of()); - } else { - this.createViewDesc = new CreateViewDesc(dbDotView, - table.getAllCols(), - null, // comment passed as table params - table.getParameters(), - table.getPartColNames(), - false,false,false, - table.getSd().getInputFormat(), - table.getSd().getOutputFormat(), - table.getSd().getSerdeInfo().getSerializationLib()); - } - - this.setViewAsReferenceText(dbName, table); - this.createViewDesc.setPartCols(table.getPartCols()); - break; - default: - throw new HiveException("Invalid table type"); - } - } - - public TYPE getDescType() { - if (table.isView() || table.isMaterializedView()) { - return TYPE.VIEW; - } - return TYPE.TABLE; - } - - public void setViewAsReferenceText(String dbName, Table table) { - String originalText = table.getViewOriginalText(); - String expandedText = table.getViewExpandedText(); - - if (!dbName.equals(table.getDbName())) { - // TODO: If the DB name doesn't match with the metadata from dump, then need to rewrite the original and expanded - // texts using new DB name. Currently it refers to the source database name. - } - - this.createViewDesc.setViewOriginalText(originalText); - this.createViewDesc.setViewExpandedText(expandedText); + TableName tableName = HiveTableName.ofNullable(table.getTableName(), dbName); + + this.createTblDesc = new CreateTableDesc(tableName, + false, // isExternal: set to false here, can be overwritten by the IMPORT stmt + false, + table.getSd().getCols(), + table.getPartitionKeys(), + table.getSd().getBucketCols(), + table.getSd().getSortCols(), + table.getSd().getNumBuckets(), + null, null, null, null, null, // these 5 delims passed as serde params + null, // comment passed as table params + table.getSd().getInputFormat(), + table.getSd().getOutputFormat(), + null, // location: set to null here, can be overwritten by the IMPORT stmt + table.getSd().getSerdeInfo().getSerializationLib(), + null, // storagehandler passed as table params + table.getSd().getSerdeInfo().getParameters(), + table.getParameters(), false, + (null == table.getSd().getSkewedInfo()) ? null : table.getSd().getSkewedInfo().getSkewedColNames(), + (null == table.getSd().getSkewedInfo()) ? null : table.getSd().getSkewedInfo().getSkewedColValues(), + null, + null, + null, + null, + null, + null, + table.getColStats(), + table.getTTable().getWriteId()); + this.createTblDesc.setStoredAsSubDirectories(table.getSd().isStoredAsSubDirectories()); } public void setReplicationSpec(ReplicationSpec replSpec) { - switch (getDescType()) { - case TABLE: - createTblDesc.setReplicationSpec(replSpec); - break; - case VIEW: - createViewDesc.setReplicationSpec(replSpec); - break; - } + createTblDesc.setReplicationSpec(replSpec); } public void setExternal(boolean isExternal) { - if (TYPE.TABLE.equals(getDescType())) { - createTblDesc.setExternal(isExternal); - } + createTblDesc.setExternal(isExternal); } public boolean isExternal() { - if (TYPE.TABLE.equals(getDescType())) { - return createTblDesc.isExternal(); - } - return false; + return createTblDesc.isExternal(); } public void setLocation(String location) { - switch (getDescType()) { - case TABLE: - createTblDesc.setLocation(location); - break; - case VIEW: - createViewDesc.setLocation(location); - break; - } + createTblDesc.setLocation(location); } public String getLocation() { - switch (getDescType()) { - case TABLE: - return createTblDesc.getLocation(); - case VIEW: - return createViewDesc.getLocation(); - } - return null; + return createTblDesc.getLocation(); } public void setTableName(TableName tableName) throws SemanticException { - switch (getDescType()) { - case TABLE: - createTblDesc.setTableName(tableName); - break; - case VIEW: - createViewDesc.setViewName(tableName.getNotEmptyDbTable()); - break; - } + createTblDesc.setTableName(tableName); } public String getTableName() throws SemanticException { - switch (getDescType()) { - case TABLE: - return createTblDesc.getTableName().getTable(); - case VIEW: - return TableName.fromString(createViewDesc.getViewName(), null, null).getTable(); - } - return null; + return createTblDesc.getTableName().getTable(); } public List getPartCols() { - switch (getDescType()) { - case TABLE: - return createTblDesc.getPartCols(); - case VIEW: - return createViewDesc.getPartCols(); - } - return null; + return createTblDesc.getPartCols(); } public List getCols() { - switch (getDescType()) { - case TABLE: - return createTblDesc.getCols(); - case VIEW: - return createViewDesc.getSchema(); - } - return null; + return createTblDesc.getCols(); } public Map getTblProps() { - switch (getDescType()) { - case TABLE: - return createTblDesc.getTblProps(); - case VIEW: - return createViewDesc.getTblProps(); - } - return null; + return createTblDesc.getTblProps(); } public String getInputFormat() { - switch (getDescType()) { - case TABLE: - return createTblDesc.getInputFormat(); - case VIEW: - return createViewDesc.getInputFormat(); - } - return null; + return createTblDesc.getInputFormat(); } public String getOutputFormat() { - switch (getDescType()) { - case TABLE: - return createTblDesc.getOutputFormat(); - case VIEW: - return createViewDesc.getOutputFormat(); - } - return null; + return createTblDesc.getOutputFormat(); } public String getSerName() { - switch (getDescType()) { - case TABLE: - return createTblDesc.getSerName(); - case VIEW: - return createViewDesc.getSerde(); - } - return null; + return createTblDesc.getSerName(); } public Map getSerdeProps() { - switch (getDescType()) { - case TABLE: - return createTblDesc.getSerdeProps(); - case VIEW: - return createViewDesc.getSerdeProps(); - } - return null; + return createTblDesc.getSerdeProps(); } public List getBucketCols() { - if (TYPE.TABLE.equals(getDescType())) { - return createTblDesc.getBucketCols(); - } - return null; + return createTblDesc.getBucketCols(); } public List getSortCols() { - if (TYPE.TABLE.equals(getDescType())) { - return createTblDesc.getSortCols(); - } - return null; + return createTblDesc.getSortCols(); } /** * @param replaceMode Determine if this CreateTable should behave like a replace-into alter instead */ public void setReplaceMode(boolean replaceMode) { - switch (getDescType()) { - case TABLE: - createTblDesc.setReplaceMode(replaceMode); - break; - case VIEW: - createViewDesc.setReplace(replaceMode); - } + createTblDesc.setReplaceMode(replaceMode); } public String getDatabaseName() { return dbName; } - public Task getCreateTableTask(Set inputs, Set outputs, - HiveConf conf) { - switch (getDescType()) { - case TABLE: - return TaskFactory.get(new DDLWork(inputs, outputs, createTblDesc), conf); - case VIEW: - return TaskFactory.get(new DDLWork(inputs, outputs, createViewDesc), conf); - } - return null; - } - - /** - * @return whether this table is actually a view - */ - public boolean isView() { - return table.isView(); - } - - public boolean isMaterializedView() { - return table.isMaterializedView(); + public Task getCreateTableTask(Set inputs, Set outputs, HiveConf conf) { + return TaskFactory.get(new DDLWork(inputs, outputs, createTblDesc), conf); } public TableType tableType() { - if (isView()) { - return TableType.VIRTUAL_VIEW; - } else if (isMaterializedView()) { - return TableType.MATERIALIZED_VIEW; - } return TableType.MANAGED_TABLE; } public Table toTable(HiveConf conf) throws Exception { - switch (getDescType()) { - case TABLE: - return createTblDesc.toTable(conf); - case VIEW: - return createViewDesc.toTable(conf); - default: - return null; - } + return createTblDesc.toTable(conf); } public void setReplWriteId(Long replWriteId) { - if (this.createTblDesc != null) { - this.createTblDesc.setReplWriteId(replWriteId); - } + this.createTblDesc.setReplWriteId(replWriteId); } public void setOwnerName(String ownerName) { - switch (getDescType()) { - case TABLE: - createTblDesc.setOwnerName(ownerName); - break; - case VIEW: - createViewDesc.setOwnerName(ownerName); - break; - default: - throw new RuntimeException("Invalid table type : " + getDescType()); - } + createTblDesc.setOwnerName(ownerName); } public Long getReplWriteId() { - if (this.createTblDesc != null) { - return this.createTblDesc.getReplWriteId(); - } - return -1L; + return this.createTblDesc.getReplWriteId(); } } diff --git ql/src/test/queries/clientnegative/export_materialized_view.q ql/src/test/queries/clientnegative/export_materialized_view.q new file mode 100644 index 0000000000..81ce8416db --- /dev/null +++ ql/src/test/queries/clientnegative/export_materialized_view.q @@ -0,0 +1,5 @@ +create table t ( a int ); + +create materialized view export_materialized_view disable rewrite as select * from t; + +export table export_materialized_view to 'ql/test/data/exports/export_materialized_view'; diff --git ql/src/test/queries/clientnegative/export_view.q ql/src/test/queries/clientnegative/export_view.q new file mode 100644 index 0000000000..086e8d9fdb --- /dev/null +++ ql/src/test/queries/clientnegative/export_view.q @@ -0,0 +1,5 @@ +create table t ( a int ); + +create view export_view as select * from t; + +export table export_view to 'ql/test/data/exports/export_view'; diff --git ql/src/test/results/clientnegative/export_materialized_view.q.out ql/src/test/results/clientnegative/export_materialized_view.q.out new file mode 100644 index 0000000000..239e41ef5f --- /dev/null +++ ql/src/test/results/clientnegative/export_materialized_view.q.out @@ -0,0 +1,19 @@ +PREHOOK: query: create table t ( a int ) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: create table t ( a int ) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: create materialized view export_materialized_view disable rewrite as select * from t +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@t +PREHOOK: Output: database:default +PREHOOK: Output: default@export_materialized_view +POSTHOOK: query: create materialized view export_materialized_view disable rewrite as select * from t +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@t +POSTHOOK: Output: database:default +POSTHOOK: Output: default@export_materialized_view +FAILED: SemanticException Views and Materialized Views can not be exported. diff --git ql/src/test/results/clientnegative/export_view.q.out ql/src/test/results/clientnegative/export_view.q.out new file mode 100644 index 0000000000..32a5c8c611 --- /dev/null +++ ql/src/test/results/clientnegative/export_view.q.out @@ -0,0 +1,20 @@ +PREHOOK: query: create table t ( a int ) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: create table t ( a int ) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: create view export_view as select * from t +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@t +PREHOOK: Output: database:default +PREHOOK: Output: default@export_view +POSTHOOK: query: create view export_view as select * from t +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@t +POSTHOOK: Output: database:default +POSTHOOK: Output: default@export_view +POSTHOOK: Lineage: export_view.a SIMPLE [(t)t.FieldSchema(name:a, type:int, comment:null), ] +FAILED: SemanticException Views and Materialized Views can not be exported.