diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java index d69696c391..76d1fb552f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java @@ -62,8 +62,8 @@ static final List AllTables = getSortedList(tableName1, tableName2); static final List AllDbs = getSortedList("default", dbName1, dbName2); - private static List filterArguments = null; - private static List filteredResults = new ArrayList(); + private static List filterArguments = new ArrayList<>(); + private static List filteredResults = new ArrayList<>(); /** * This factory creates a mocked HiveAuthorizer class. The mocked class is @@ -78,7 +78,7 @@ public List filterListCmdObjects(List listObjs, HiveAuthzContext context) throws HiveAuthzPluginException, HiveAccessControlException { // capture arguments in static - filterArguments = listObjs; + filterArguments.addAll(listObjs); // return static variable with results, if it is set to some set of // values // otherwise return the arguments @@ -137,7 +137,7 @@ public static void beforeTest() throws Exception { @Before public void setup() { - filterArguments = null; + filterArguments.clear(); filteredResults.clear(); } diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index f51ff0620c..7259bd5d27 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -557,6 +557,7 @@ minillaplocal.query.files=\ materialized_view_create_rewrite_multi_db.q,\ materialized_view_create_rewrite_rebuild_dummy.q,\ materialized_view_create_rewrite_time_window.q,\ + materialized_view_create_rewrite_time_window_2.q,\ materialized_view_create_rewrite.q,\ materialized_view_create_rewrite_2.q,\ materialized_view_create_rewrite_3.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 939ef360c2..accd7f1324 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -49,6 +49,7 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -64,6 +65,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.ValidTxnWriteIdList; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.conf.Constants; @@ -2776,6 +2778,7 @@ private int showDatabases(Hive db, ShowDatabasesDesc showDatabasesDesc) throws H private int showTablesOrViews(Hive db, ShowTablesDesc showDesc) throws HiveException { // get the tables/views for the desired pattern - populate the output stream List tablesOrViews = null; + List materializedViews = null; String dbName = showDesc.getDbName(); String pattern = showDesc.getPattern(); // if null, all tables/views are returned @@ -2787,8 +2790,21 @@ private int showTablesOrViews(Hive db, ShowTablesDesc showDesc) throws HiveExcep } LOG.debug("pattern: {}", pattern); - tablesOrViews = db.getTablesByType(dbName, pattern, type); - LOG.debug("Found {} tables/view(s) matching the SHOW TABLES/VIEWS statement.", tablesOrViews.size()); + if (type == null) { + tablesOrViews = new ArrayList<>(); + tablesOrViews.addAll(db.getTablesByType(dbName, pattern, TableType.MANAGED_TABLE)); + tablesOrViews.addAll(db.getTablesByType(dbName, pattern, TableType.EXTERNAL_TABLE)); + LOG.debug("Found {} table(s) matching the SHOW TABLES statement.", tablesOrViews.size()); + } else if (type == TableType.MATERIALIZED_VIEW) { + materializedViews = new ArrayList<>(); + materializedViews.addAll(db.getAllMaterializedViewObjects(dbName)); + LOG.debug("Found {} materialized view(s) matching the SHOW MATERIALIZED VIEWS statement.", materializedViews.size()); + } else if (type == TableType.VIRTUAL_VIEW) { + tablesOrViews = db.getTablesByType(dbName, pattern, type); + LOG.debug("Found {} view(s) matching the SHOW VIEWS statement.", tablesOrViews.size()); + } else { + throw new HiveException("Option not recognized in SHOW TABLES/VIEWS/MATERIALIZED VIEWS"); + } // write the results in the file DataOutputStream outStream = null; @@ -2796,11 +2812,15 @@ private int showTablesOrViews(Hive db, ShowTablesDesc showDesc) throws HiveExcep Path resFile = new Path(resultsFile); FileSystem fs = resFile.getFileSystem(conf); outStream = fs.create(resFile); - - SortedSet sortedSet = new TreeSet(tablesOrViews); - formatter.showTables(outStream, sortedSet); + // Sort by name and print + if (tablesOrViews != null) { + SortedSet sortedSet = new TreeSet(tablesOrViews); + formatter.showTables(outStream, sortedSet); + } else { + Collections.sort(materializedViews, Comparator.comparing(Table::getTableName)); + formatter.showMaterializedViews(outStream, materializedViews); + } outStream.close(); - outStream = null; } catch (Exception e) { throw new HiveException(e, ErrorMsg.GENERIC_ERROR, "in database" + dbName); } finally { @@ -3735,13 +3755,28 @@ private int describeTable(Hive db, DescTableDesc descTbl) throws HiveException, storageHandlerInfo = db.getStorageHandlerInfo(tbl); } fixDecimalColumnTypeName(cols); + // Information for materialized views + if (tbl.isMaterializedView()) { + final String validTxnsList = db.getConf().get(ValidTxnList.VALID_TXNS_KEY); + if (validTxnsList != null) { + final List tablesUsed = + new ArrayList<>(tbl.getCreationMetadata().getTablesUsed()); + final ValidTxnWriteIdList currentTxnWriteIds = + SessionState.get().getTxnMgr().getValidWriteIds(tablesUsed, validTxnsList); + final long defaultTimeWindow = + HiveConf.getTimeVar(db.getConf(), HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW, + TimeUnit.MILLISECONDS); + tbl.setOutdatedForRewriting(Hive.isOutdatedMaterializedView(tbl, + currentTxnWriteIds, defaultTimeWindow, tablesUsed, false)); + } + } // In case the query is served by HiveServer2, don't pad it with spaces, // as HiveServer2 output is consumed by JDBC/ODBC clients. boolean isOutputPadded = !SessionState.get().isHiveServerQuery(); formatter.describeTable(outStream, colPath, tableName, tbl, part, - cols, descTbl.isFormatted(), descTbl.isExt(), - isOutputPadded, colStats, - pkInfo, fkInfo, ukInfo, nnInfo, dInfo, cInfo, storageHandlerInfo); + cols, descTbl.isFormatted(), descTbl.isExt(), isOutputPadded, + colStats, pkInfo, fkInfo, ukInfo, nnInfo, dInfo, cInfo, + storageHandlerInfo); LOG.debug("DDLTask: written data for {}", tableName); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 239a606fdb..2a360433f2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1425,82 +1425,21 @@ public Table apply(org.apache.hadoop.hive.metastore.api.Table table) { HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_INCREMENTAL); final boolean tryIncrementalRebuild = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REBUILD_INCREMENTAL); - final long defaultDiff = + final long defaultTimeWindow = HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW, TimeUnit.MILLISECONDS); - final long currentTime = System.currentTimeMillis(); try { // Final result List result = new ArrayList<>(); List
materializedViewTables = getTableObjects(dbName, materializedViewNames); for (Table materializedViewTable : materializedViewTables) { - // Check if materialization defined its own invalidation time window - String timeWindowString = materializedViewTable.getProperty(MATERIALIZED_VIEW_REWRITING_TIME_WINDOW); - long diff = org.apache.commons.lang.StringUtils.isEmpty(timeWindowString) ? defaultDiff : - HiveConf.toTime(timeWindowString, - HiveConf.getDefaultTimeUnit(HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW), - TimeUnit.MILLISECONDS); - CreationMetadata creationMetadata = materializedViewTable.getCreationMetadata(); - boolean outdated = false; - if (diff < 0L) { - // We only consider the materialized view to be outdated if forceOutdated = true, i.e., - // if it is a rebuild. Otherwise, it passed the test and we use it as it is. - outdated = forceMVContentsUpToDate; - } else { - // Check whether the materialized view is invalidated - if (forceMVContentsUpToDate || diff == 0L || creationMetadata.getMaterializationTime() < currentTime - diff) { - if (currentTxnWriteIds == null) { - LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + - " ignored for rewriting as we could not obtain current txn ids"); - continue; - } - if (creationMetadata.getValidTxnList() == null || - creationMetadata.getValidTxnList().isEmpty()) { - LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + - " ignored for rewriting as we could not obtain materialization txn ids"); - continue; - } - boolean ignore = false; - ValidTxnWriteIdList mvTxnWriteIds = new ValidTxnWriteIdList( - creationMetadata.getValidTxnList()); - for (String qName : tablesUsed) { - // Note. If the materialized view does not contain a table that is contained in the query, - // we do not need to check whether that specific table is outdated or not. If a rewriting - // is produced in those cases, it is because that additional table is joined with the - // existing tables with an append-columns only join, i.e., PK-FK + not null. - if (!creationMetadata.getTablesUsed().contains(qName)) { - continue; - } - ValidWriteIdList tableCurrentWriteIds = currentTxnWriteIds.getTableValidWriteIdList(qName); - if (tableCurrentWriteIds == null) { - // Uses non-transactional table, cannot be considered - LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + - " ignored for rewriting as it is outdated and cannot be considered for " + - " rewriting because it uses non-transactional table " + qName); - ignore = true; - break; - } - ValidWriteIdList tableWriteIds = mvTxnWriteIds.getTableValidWriteIdList(qName); - if (tableWriteIds == null) { - // This should not happen, but we ignore for safety - LOG.warn("Materialized view " + materializedViewTable.getFullyQualifiedName() + - " ignored for rewriting as details about txn ids for table " + qName + - " could not be found in " + mvTxnWriteIds); - ignore = true; - break; - } - if (!outdated && !TxnIdUtils.checkEquivalentWriteIds(tableCurrentWriteIds, tableWriteIds)) { - LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + - " contents are outdated"); - outdated = true; - } - } - if (ignore) { - continue; - } - } + final Boolean outdated = isOutdatedMaterializedView(materializedViewTable, currentTxnWriteIds, + defaultTimeWindow, tablesUsed, forceMVContentsUpToDate); + if (outdated == null) { + continue; } + final CreationMetadata creationMetadata = materializedViewTable.getCreationMetadata(); if (outdated) { // The MV is outdated, see whether we should consider it for rewriting or not boolean ignore = false; @@ -1586,6 +1525,82 @@ public Table apply(org.apache.hadoop.hive.metastore.api.Table table) { } } + /** + * Utility method that returns whether a materialized view is outdated (true), not outdated + * (false), or it cannot be determined (null). The latest case may happen e.g. when the + * materialized view definition uses external tables. + */ + public static Boolean isOutdatedMaterializedView(Table materializedViewTable, final ValidTxnWriteIdList currentTxnWriteIds, + long defaultTimeWindow, List tablesUsed, boolean forceMVContentsUpToDate) { + // Check if materialization defined its own invalidation time window + String timeWindowString = materializedViewTable.getProperty(MATERIALIZED_VIEW_REWRITING_TIME_WINDOW); + long timeWindow = org.apache.commons.lang.StringUtils.isEmpty(timeWindowString) ? defaultTimeWindow : + HiveConf.toTime(timeWindowString, + HiveConf.getDefaultTimeUnit(HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW), + TimeUnit.MILLISECONDS); + CreationMetadata creationMetadata = materializedViewTable.getCreationMetadata(); + boolean outdated = false; + if (timeWindow < 0L) { + // We only consider the materialized view to be outdated if forceOutdated = true, i.e., + // if it is a rebuild. Otherwise, it passed the test and we use it as it is. + outdated = forceMVContentsUpToDate; + } else { + // Check whether the materialized view is invalidated + if (forceMVContentsUpToDate || timeWindow == 0L || creationMetadata.getMaterializationTime() < System.currentTimeMillis() - timeWindow) { + if (currentTxnWriteIds == null) { + LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + + " ignored for rewriting as we could not obtain current txn ids"); + return null; + } + if (creationMetadata.getValidTxnList() == null || + creationMetadata.getValidTxnList().isEmpty()) { + LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + + " ignored for rewriting as we could not obtain materialization txn ids"); + return null; + } + boolean ignore = false; + ValidTxnWriteIdList mvTxnWriteIds = new ValidTxnWriteIdList( + creationMetadata.getValidTxnList()); + for (String qName : tablesUsed) { + // Note. If the materialized view does not contain a table that is contained in the query, + // we do not need to check whether that specific table is outdated or not. If a rewriting + // is produced in those cases, it is because that additional table is joined with the + // existing tables with an append-columns only join, i.e., PK-FK + not null. + if (!creationMetadata.getTablesUsed().contains(qName)) { + continue; + } + ValidWriteIdList tableCurrentWriteIds = currentTxnWriteIds.getTableValidWriteIdList(qName); + if (tableCurrentWriteIds == null) { + // Uses non-transactional table, cannot be considered + LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + + " ignored for rewriting as it is outdated and cannot be considered for " + + " rewriting because it uses non-transactional table " + qName); + ignore = true; + break; + } + ValidWriteIdList tableWriteIds = mvTxnWriteIds.getTableValidWriteIdList(qName); + if (tableWriteIds == null) { + // This should not happen, but we ignore for safety + LOG.warn("Materialized view " + materializedViewTable.getFullyQualifiedName() + + " ignored for rewriting as details about txn ids for table " + qName + + " could not be found in " + mvTxnWriteIds); + ignore = true; + break; + } + if (!outdated && !TxnIdUtils.checkEquivalentWriteIds(tableCurrentWriteIds, tableWriteIds)) { + LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + + " contents are outdated"); + outdated = true; + } + } + if (ignore) { + return null; + } + } + } + return outdated; + } + /** * Method to enrich the materialization query contained in the input with * its invalidation. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java index 3240f2d315..a2b57fb646 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java @@ -26,6 +26,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -40,6 +41,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; @@ -244,6 +246,46 @@ public void truncateTable(String dbName, String tableName, return tableNames; } + @Override + public List getTables(String dbname, String tablePattern, TableType tableType) throws MetaException { + List tableNames = super.getTables(dbname, tablePattern, tableType); + + if (tableType == TableType.MANAGED_TABLE || tableType == TableType.EXTERNAL_TABLE) { + // May need to merge with list of temp tables + dbname = dbname.toLowerCase(); + tablePattern = tablePattern.toLowerCase(); + Map tables = getTempTablesForDatabase(dbname, tablePattern); + if (tables == null || tables.size() == 0) { + return tableNames; + } + tablePattern = tablePattern.replaceAll("\\*", ".*"); + Pattern pattern = Pattern.compile(tablePattern); + Matcher matcher = pattern.matcher(""); + Set combinedTableNames = new HashSet(); + combinedTableNames.addAll(tableNames); + for (Entry tableData : tables.entrySet()) { + matcher.reset(tableData.getKey()); + if (matcher.matches()) { + if (tableData.getValue().getTableType() == tableType) { + // If tableType is the same that we are requesting, + // add table the the list + combinedTableNames.add(tableData.getKey()); + } else { + // If tableType is not the same that we are requesting, + // remove it in case it was added before, as temp table + // overrides original table + combinedTableNames.remove(tableData.getKey()); + } + } + } + // Combine/sort temp and normal table results + tableNames = new ArrayList<>(combinedTableNames); + Collections.sort(tableNames); + } + + return tableNames; + } + @Override public List getTableMeta(String dbPatterns, String tablePatterns, List tableTypes) throws MetaException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 03b0269441..9e0cea7af6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -105,6 +105,10 @@ private transient boolean materializedTable; + /** Note: This is set only for describe table purposes, it cannot be used to verify whether + * a materialization is up-to-date or not. */ + private transient Boolean outdatedForRewritingMaterializedView; + /** * Used only for serialization. */ @@ -1091,4 +1095,14 @@ public boolean hasDeserializer() { public String getCatalogName() { return this.tTable.getCatName(); } + + public void setOutdatedForRewriting(Boolean validForRewritingMaterializedView) { + this.outdatedForRewritingMaterializedView = validForRewritingMaterializedView; + } + + /** Note: This is set only for describe table purposes, it cannot be used to verify whether + * a materialization is up-to-date or not. */ + public Boolean isOutdatedForRewriting() { + return outdatedForRewritingMaterializedView; + } }; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java index df0a2370a9..4c4f9c1872 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java @@ -28,9 +28,11 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.ql.session.SessionState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.FileStatus; @@ -57,6 +59,8 @@ import org.codehaus.jackson.JsonGenerator; import org.codehaus.jackson.map.ObjectMapper; +import static org.apache.hadoop.hive.conf.Constants.MATERIALIZED_VIEW_REWRITING_TIME_WINDOW; + /** * Format table and index information for machine readability using * json. @@ -108,6 +112,50 @@ public void showTables(DataOutputStream out, Set tables) asJson(out, MapBuilder.create().put("tables", tables).build()); } + /** + * Show a list of materialized views. + */ + @Override + public void showMaterializedViews(DataOutputStream out, List
materializedViews) + throws HiveException { + if (materializedViews.isEmpty()) { + // Nothing to do + return; + } + + MapBuilder builder = MapBuilder.create(); + ArrayList> res = new ArrayList>(); + for (Table mv : materializedViews) { + final String mvName = mv.getTableName(); + final String rewriteEnabled = mv.isRewriteEnabled() ? "Yes" : "No"; + // Currently, we only support manual refresh + // TODO: Update whenever we have other modes + final String refreshMode = "Manual refresh"; + final String timeWindowString = mv.getProperty(MATERIALIZED_VIEW_REWRITING_TIME_WINDOW); + final String mode; + if (!org.apache.commons.lang.StringUtils.isEmpty(timeWindowString)) { + long time = HiveConf.toTime(timeWindowString, + HiveConf.getDefaultTimeUnit(HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW), + TimeUnit.MINUTES); + if (time > 0L) { + mode = refreshMode + " (Valid for " + time + "min)"; + } else if (time == 0L) { + mode = refreshMode + " (Valid until source tables modified)"; + } else { + mode = refreshMode + " (Valid always)"; + } + } else { + mode = refreshMode; + } + res.add(builder + .put("MV Name", mvName) + .put("Rewriting Enabled", rewriteEnabled) + .put("Mode", mode) + .build()); + } + asJson(out, builder.put("materialized views", res).build()); + } + /** * Describe table. */ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java index 36cd46aa43..4180dc471d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java @@ -360,7 +360,7 @@ public static String getTableInformation(Table table, boolean isOutputPadded) { getStorageDescriptorInfo(tableInfo, table.getTTable().getSd()); if (table.isView() || table.isMaterializedView()) { - tableInfo.append(LINE_DELIM).append("# View Information").append(LINE_DELIM); + tableInfo.append(LINE_DELIM).append(table.isView() ? "# View Information" : "# Materialized View Information").append(LINE_DELIM); getViewInfo(tableInfo, table); } @@ -368,9 +368,13 @@ public static String getTableInformation(Table table, boolean isOutputPadded) { } private static void getViewInfo(StringBuilder tableInfo, Table tbl) { - formatOutput("View Original Text:", tbl.getViewOriginalText(), tableInfo); - formatOutput("View Expanded Text:", tbl.getViewExpandedText(), tableInfo); - formatOutput("View Rewrite Enabled:", tbl.isRewriteEnabled() ? "Yes" : "No", tableInfo); + formatOutput("Original Query:", tbl.getViewOriginalText(), tableInfo); + formatOutput("Expanded Query:", tbl.getViewExpandedText(), tableInfo); + if (tbl.isMaterializedView()) { + formatOutput("Rewrite Enabled:", tbl.isRewriteEnabled() ? "Yes" : "No", tableInfo); + formatOutput("Outdated for Rewriting:", tbl.isOutdatedForRewriting() == null ? "Unknown" + : tbl.isOutdatedForRewriting() ? "Yes" : "No", tableInfo); + } } private static void getStorageDescriptorInfo(StringBuilder tableInfo, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java index d15016c5d4..936a80870d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java @@ -20,10 +20,12 @@ import java.io.DataOutputStream; import java.io.OutputStream; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; +import javax.annotation.Nullable; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -68,6 +70,12 @@ public void error(OutputStream out, String errorMessage, int errorCode, String s public void showTables(DataOutputStream out, Set tables) throws HiveException; + /** + * Show a list of materialized views. + */ + public void showMaterializedViews(DataOutputStream out, List
materializedViews) + throws HiveException; + /** * Describe table. * @param out diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java index 705365b74c..2ff1d94e74 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java @@ -27,9 +27,12 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.metadata.StorageHandlerInfo; +import org.apache.hadoop.hive.ql.plan.DescTableDesc; import org.apache.hive.common.util.HiveStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,6 +61,8 @@ import org.apache.hadoop.hive.ql.metadata.UniqueConstraint; import org.apache.hadoop.hive.ql.session.SessionState; +import static org.apache.hadoop.hive.conf.Constants.MATERIALIZED_VIEW_REWRITING_TIME_WINDOW; + /** * Format table and index information for human readability using * simple lines of text. @@ -106,14 +111,14 @@ public void error(OutputStream out, String errorMessage, int errorCode, String s } catch (Exception e) { throw new HiveException(e); } - } + } + /** * Show a list of tables. */ @Override public void showTables(DataOutputStream out, Set tables) - throws HiveException - { + throws HiveException { Iterator iterTbls = tables.iterator(); try { @@ -125,7 +130,54 @@ public void showTables(DataOutputStream out, Set tables) } catch (IOException e) { throw new HiveException(e); } + } + + /** + * Show a list of materialized views. + */ + @Override + public void showMaterializedViews(DataOutputStream out, List
materializedViews) + throws HiveException { + if (materializedViews.isEmpty()) { + // Nothing to do + return; + } + + try { + TextMetaDataTable mdt = new TextMetaDataTable(); + mdt.addRow("# MV Name", "Rewriting Enabled", "Mode"); + for (Table mv : materializedViews) { + final String mvName = mv.getTableName(); + final String rewriteEnabled = mv.isRewriteEnabled() ? "Yes" : "No"; + // Currently, we only support manual refresh + // TODO: Update whenever we have other modes + final String refreshMode = "Manual refresh"; + final String timeWindowString = mv.getProperty(MATERIALIZED_VIEW_REWRITING_TIME_WINDOW); + final String mode; + if (!org.apache.commons.lang.StringUtils.isEmpty(timeWindowString)) { + long time = HiveConf.toTime(timeWindowString, + HiveConf.getDefaultTimeUnit(HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW), + TimeUnit.MINUTES); + if (time > 0L) { + mode = refreshMode + " (Valid for " + time + "min)"; + } else if (time == 0L) { + mode = refreshMode + " (Valid until source tables modified)"; + } else { + mode = refreshMode + " (Valid always)"; + } + } else { + mode = refreshMode; + } + mdt.addRow(mvName, rewriteEnabled, mode); } + // In case the query is served by HiveServer2, don't pad it with spaces, + // as HiveServer2 output is consumed by JDBC/ODBC clients. + out.write(mdt.renderTable(!SessionState.get().isHiveServerQuery()).getBytes("UTF-8")); + out.write(terminator); + } catch (IOException e) { + throw new HiveException(e); + } + } @Override public void describeTable(DataOutputStream outStream, String colPath, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowTablesDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowTablesDesc.java index bf80345623..1c162526b1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowTablesDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowTablesDesc.java @@ -32,25 +32,38 @@ @Explain(displayName = "Show Tables", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public class ShowTablesDesc extends DDLDesc implements Serializable { private static final long serialVersionUID = 1L; - String pattern; - String dbName; - String resFile; - TableType type; + /** * table name for the result of show tables. */ private static final String table = "show"; + + /** + * thrift ddl for the result of show tables. + */ + private static final String TABLES_VIEWS_SCHEMA = "tab_name#string"; + /** * thrift ddl for the result of show tables. */ - private static final String schema = "tab_name#string"; + private static final String MATERIALIZED_VIEWS_SCHEMA = + "mv_name,rewrite_enabled,mode#string:string:string"; + + + String pattern; + String dbName; + String resFile; + TableType type; public String getTable() { return table; } public String getSchema() { - return schema; + if (type != null && type == TableType.MATERIALIZED_VIEW) { + return MATERIALIZED_VIEWS_SCHEMA; + } + return TABLES_VIEWS_SCHEMA; } public ShowTablesDesc() { diff --git a/ql/src/test/queries/clientpositive/create_view.q b/ql/src/test/queries/clientpositive/create_view.q index 3d66cbce8e..dce2866f8d 100644 --- a/ql/src/test/queries/clientpositive/create_view.q +++ b/ql/src/test/queries/clientpositive/create_view.q @@ -42,6 +42,7 @@ EXPLAIN SELECT * from view2 where key=18; SHOW TABLES 'view.*'; +SHOW VIEWS 'view.*'; DESCRIBE view1; DESCRIBE EXTENDED view1; DESCRIBE FORMATTED view1; diff --git a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_time_window_2.q b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_time_window_2.q new file mode 100644 index 0000000000..6873673a55 --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_time_window_2.q @@ -0,0 +1,45 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; + +create table cmv_basetable_n100 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true'); +insert into cmv_basetable_n100 values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1); +analyze table cmv_basetable_n100 compute statistics for columns; + +create table cmv_basetable_2_n100 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true'); +insert into cmv_basetable_2_n100 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3); +analyze table cmv_basetable_2_n100 compute statistics for columns; + +-- CREATE MATERIALIZED VIEW +CREATE MATERIALIZED VIEW cmv_mat_view_n300 AS + SELECT cmv_basetable_n100.a, cmv_basetable_2_n100.c + FROM cmv_basetable_n100 JOIN cmv_basetable_2_n100 ON (cmv_basetable_n100.a = cmv_basetable_2_n100.a) + WHERE cmv_basetable_2_n100.c > 10.0 + GROUP BY cmv_basetable_n100.a, cmv_basetable_2_n100.c; + +-- OUTDATED: NO +DESCRIBE FORMATTED cmv_mat_view_n300; + +insert into cmv_basetable_2_n100 values + (3, 'charlie', 15.8, 1); +analyze table cmv_basetable_2_n100 compute statistics for columns; + +-- OUTDATED: YES +DESCRIBE FORMATTED cmv_mat_view_n300; + +-- REBUILD +ALTER MATERIALIZED VIEW cmv_mat_view_n300 REBUILD; + +-- OUTDATED: NO +DESCRIBE FORMATTED cmv_mat_view_n300; + +drop materialized view cmv_mat_view_n300; +drop table cmv_basetable_n100; +drop table cmv_basetable_2_n100; diff --git a/ql/src/test/queries/clientpositive/show_materialized_views.q b/ql/src/test/queries/clientpositive/show_materialized_views.q index f8d49d684d..7c91b2eb5e 100644 --- a/ql/src/test/queries/clientpositive/show_materialized_views.q +++ b/ql/src/test/queries/clientpositive/show_materialized_views.q @@ -11,9 +11,11 @@ CREATE TABLE shtb_test1(KEY INT, VALUE STRING) PARTITIONED BY(ds STRING) STORED AS ORC TBLPROPERTIES ('transactional'='true'); CREATE MATERIALIZED VIEW shtb_test1_view1 DISABLE REWRITE AS SELECT * FROM shtb_test1 where KEY > 1000 and KEY < 2000; -CREATE MATERIALIZED VIEW shtb_test1_view2 AS +CREATE MATERIALIZED VIEW shtb_test1_view2 +TBLPROPERTIES ('rewriting.time.window' = '-1min') AS SELECT * FROM shtb_test1 where KEY > 100 and KEY < 200; -CREATE MATERIALIZED VIEW shtb_full_view2 AS +CREATE MATERIALIZED VIEW shtb_full_view2 +TBLPROPERTIES ('rewriting.time.window' = '5min') AS SELECT * FROM shtb_test1; USE test2; @@ -42,16 +44,22 @@ SHOW MATERIALIZED VIEWS FROM test2; SHOW MATERIALIZED VIEWS IN test1; SHOW MATERIALIZED VIEWS IN default; SHOW MATERIALIZED VIEWS IN test1 "shtb_test_*"; +DESCRIBE FORMATTED test1.shtb_full_view2; +DESCRIBE FORMATTED test1.shtb_test1_view1; +DESCRIBE FORMATTED test1.shtb_test1_view2; SHOW MATERIALIZED VIEWS IN test2 LIKE "nomatch"; -- SHOW MATERIALIZED VIEWS from a database with a name that requires escaping CREATE DATABASE `database`; USE `database`; -CREATE TABLE foo_n0(a INT); -CREATE VIEW fooview AS +CREATE TABLE foo_n0(a INT) +STORED AS ORC TBLPROPERTIES ('transactional'='true'); +CREATE MATERIALIZED VIEW fooview +TBLPROPERTIES ('rewriting.time.window' = '0min') AS SELECT * FROM foo_n0; USE default; SHOW MATERIALIZED VIEWS FROM `database` LIKE "fooview"; +DESCRIBE FORMATTED `database`.`fooview`; DROP MATERIALIZED VIEW fooview; DROP TABLE foo_n0; diff --git a/ql/src/test/queries/clientpositive/temp_table.q b/ql/src/test/queries/clientpositive/temp_table.q index 4990223b7a..f462f70da6 100644 --- a/ql/src/test/queries/clientpositive/temp_table.q +++ b/ql/src/test/queries/clientpositive/temp_table.q @@ -8,7 +8,7 @@ EXPLAIN CREATE TEMPORARY TABLE bar AS SELECT * FROM src WHERE key % 2 = 1; CREATE TEMPORARY TABLE bar AS SELECT * FROM src WHERE key % 2 = 1; DESCRIBE foo; -DESCRIBE bar; +DESCRIBE FORMATTED bar; explain select * from foo order by key limit 10; select * from foo order by key limit 10; diff --git a/ql/src/test/results/clientpositive/create_view.q.out b/ql/src/test/results/clientpositive/create_view.q.out index b5b7bfb13a..d2c516fdfe 100644 --- a/ql/src/test/results/clientpositive/create_view.q.out +++ b/ql/src/test/results/clientpositive/create_view.q.out @@ -221,6 +221,10 @@ PREHOOK: Input: database:default POSTHOOK: query: SHOW TABLES 'view.*' POSTHOOK: type: SHOWTABLES POSTHOOK: Input: database:default +PREHOOK: query: SHOW VIEWS 'view.*' +PREHOOK: type: SHOWVIEWS +POSTHOOK: query: SHOW VIEWS 'view.*' +POSTHOOK: type: SHOWVIEWS view1 view2 view3 @@ -268,9 +272,8 @@ Bucket Columns: [] Sort Columns: [] # View Information -View Original Text: SELECT value FROM src WHERE key=86 -View Expanded Text: SELECT `src`.`value` FROM `default`.`src` WHERE `src`.`key`=86 -View Rewrite Enabled: No +Original Query: SELECT value FROM src WHERE key=86 +Expanded Query: SELECT `src`.`value` FROM `default`.`src` WHERE `src`.`key`=86 PREHOOK: query: DESCRIBE view2 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view2 @@ -318,9 +321,8 @@ Bucket Columns: [] Sort Columns: [] # View Information -View Original Text: SELECT * FROM src -View Expanded Text: SELECT `src`.`key`, `src`.`value` FROM `default`.`src` -View Rewrite Enabled: No +Original Query: SELECT * FROM src +Expanded Query: SELECT `src`.`key`, `src`.`value` FROM `default`.`src` PREHOOK: query: DESCRIBE view3 PREHOOK: type: DESCTABLE PREHOOK: Input: default@view3 @@ -366,9 +368,8 @@ Bucket Columns: [] Sort Columns: [] # View Information -View Original Text: SELECT upper(value) FROM src WHERE key=86 -View Expanded Text: SELECT `_c0` AS `valoo` FROM (SELECT upper(`src`.`value`) FROM `default`.`src` WHERE `src`.`key`=86) `default.view3` -View Rewrite Enabled: No +Original Query: SELECT upper(value) FROM src WHERE key=86 +Expanded Query: SELECT `_c0` AS `valoo` FROM (SELECT upper(`src`.`value`) FROM `default`.`src` WHERE `src`.`key`=86) `default.view3` PREHOOK: query: ALTER VIEW view3 SET TBLPROPERTIES ("biggest" = "loser") PREHOOK: type: ALTERVIEW_PROPERTIES PREHOOK: Input: default@view3 @@ -416,9 +417,8 @@ Bucket Columns: [] Sort Columns: [] # View Information -View Original Text: SELECT upper(value) FROM src WHERE key=86 -View Expanded Text: SELECT `_c0` AS `valoo` FROM (SELECT upper(`src`.`value`) FROM `default`.`src` WHERE `src`.`key`=86) `default.view3` -View Rewrite Enabled: No +Original Query: SELECT upper(value) FROM src WHERE key=86 +Expanded Query: SELECT `_c0` AS `valoo` FROM (SELECT upper(`src`.`value`) FROM `default`.`src` WHERE `src`.`key`=86) `default.view3` PREHOOK: query: CREATE TABLE table1_n4 (key int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -758,11 +758,10 @@ Bucket Columns: [] Sort Columns: [] # View Information -View Original Text: SELECT test_translate('abc', 'a', 'b') +Original Query: SELECT test_translate('abc', 'a', 'b') FROM table1_n4 -View Expanded Text: SELECT `_c0` AS `c` FROM (SELECT `test_translate`('abc', 'a', 'b') +Expanded Query: SELECT `_c0` AS `c` FROM (SELECT `test_translate`('abc', 'a', 'b') FROM `default`.`table1_n4`) `default.view8` -View Rewrite Enabled: No PREHOOK: query: SELECT * FROM view8 PREHOOK: type: QUERY PREHOOK: Input: default@table1_n4 @@ -834,11 +833,10 @@ Bucket Columns: [] Sort Columns: [] # View Information -View Original Text: SELECT test_max(length(value)) +Original Query: SELECT test_max(length(value)) FROM src -View Expanded Text: SELECT `_c0` AS `m` FROM (SELECT `test_max`(length(`src`.`value`)) +Expanded Query: SELECT `_c0` AS `m` FROM (SELECT `test_max`(length(`src`.`value`)) FROM `default`.`src`) `default.view9` -View Rewrite Enabled: No PREHOOK: query: SELECT * FROM view9 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -910,11 +908,10 @@ Bucket Columns: [] Sort Columns: [] # View Information -View Original Text: SELECT test_max(length(value)) +Original Query: SELECT test_max(length(value)) FROM src -View Expanded Text: SELECT `_c0` AS `m` FROM (SELECT `test_max`(length(`src`.`value`)) +Expanded Query: SELECT `_c0` AS `m` FROM (SELECT `test_max`(length(`src`.`value`)) FROM `default`.`src`) `default.view9` -View Rewrite Enabled: No PREHOOK: query: SELECT * FROM view9 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -979,9 +976,8 @@ Bucket Columns: [] Sort Columns: [] # View Information -View Original Text: SELECT slurp.* FROM (SELECT * FROM src WHERE key=86) slurp -View Expanded Text: SELECT `slurp`.`key`, `slurp`.`value` FROM (SELECT `src`.`key`, `src`.`value` FROM `default`.`src` WHERE `src`.`key`=86) `slurp` -View Rewrite Enabled: No +Original Query: SELECT slurp.* FROM (SELECT * FROM src WHERE key=86) slurp +Expanded Query: SELECT `slurp`.`key`, `slurp`.`value` FROM (SELECT `src`.`key`, `src`.`value` FROM `default`.`src` WHERE `src`.`key`=86) `slurp` PREHOOK: query: SELECT * FROM view10 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -1053,11 +1049,10 @@ Bucket Columns: [] Sort Columns: [] # View Information -View Original Text: SELECT test_explode(array(1,2,3)) AS (boom) +Original Query: SELECT test_explode(array(1,2,3)) AS (boom) FROM table1_n4 -View Expanded Text: SELECT `test_explode`(array(1,2,3)) AS (`boom`) +Expanded Query: SELECT `test_explode`(array(1,2,3)) AS (`boom`) FROM `default`.`table1_n4` -View Rewrite Enabled: No PREHOOK: query: SELECT * FROM view11 PREHOOK: type: QUERY PREHOOK: Input: default@table1_n4 @@ -1127,9 +1122,8 @@ Bucket Columns: [] Sort Columns: [] # View Information -View Original Text: SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol -View Expanded Text: SELECT `src`.`key`, `src`.`value`, `mytable`.`mycol` FROM `default`.`src` LATERAL VIEW explode(array(1,2,3)) `myTable` AS `myCol` -View Rewrite Enabled: No +Original Query: SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +Expanded Query: SELECT `src`.`key`, `src`.`value`, `mytable`.`mycol` FROM `default`.`src` LATERAL VIEW explode(array(1,2,3)) `myTable` AS `myCol` PREHOOK: query: SELECT * FROM view12 ORDER BY key ASC, myCol ASC LIMIT 1 PREHOOK: type: QUERY @@ -1208,11 +1202,10 @@ Bucket Columns: [] Sort Columns: [] # View Information -View Original Text: SELECT s.key +Original Query: SELECT s.key FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 ON key) s -View Expanded Text: SELECT `s`.`key` +Expanded Query: SELECT `s`.`key` FROM `default`.`srcbucket` TABLESAMPLE (BUCKET 1 OUT OF 5 ON `key`) `s` -View Rewrite Enabled: No PREHOOK: query: SELECT * FROM view13 ORDER BY key LIMIT 12 PREHOOK: type: QUERY @@ -1314,7 +1307,7 @@ Bucket Columns: [] Sort Columns: [] # View Information -View Original Text: SELECT unionsrc1.key as k1, unionsrc1.value as v1, +Original Query: SELECT unionsrc1.key as k1, unionsrc1.value as v1, unionsrc2.key as k2, unionsrc2.value as v2 FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL @@ -1324,7 +1317,7 @@ View Original Text: SELECT unionsrc1.key as k1, unionsrc1.value as v1, UNION ALL select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 ON (unionsrc1.key = unionsrc2.key) -View Expanded Text: SELECT `unionsrc1`.`key` as `k1`, `unionsrc1`.`value` as `v1`, +Expanded Query: SELECT `unionsrc1`.`key` as `k1`, `unionsrc1`.`value` as `v1`, `unionsrc2`.`key` as `k2`, `unionsrc2`.`value` as `v2` FROM (select 'tst1' as `key`, cast(count(1) as string) as `value` from `default`.`src` `s1` UNION ALL @@ -1334,7 +1327,6 @@ View Expanded Text: SELECT `unionsrc1`.`key` as `k1`, `unionsrc1`.`value` as `v UNION ALL select `s4`.`key` as `key`, `s4`.`value` as `value` from `default`.`src` `s4` where `s4`.`key` < 10) `unionsrc2` ON (`unionsrc1`.`key` = `unionsrc2`.`key`) -View Rewrite Enabled: No PREHOOK: query: SELECT * FROM view14 ORDER BY k1 PREHOOK: type: QUERY @@ -1427,13 +1419,12 @@ Bucket Columns: [] Sort Columns: [] # View Information -View Original Text: SELECT key,COUNT(value) AS value_count +Original Query: SELECT key,COUNT(value) AS value_count FROM src GROUP BY key -View Expanded Text: SELECT `src`.`key`,COUNT(`src`.`value`) AS `value_count` +Expanded Query: SELECT `src`.`key`,COUNT(`src`.`value`) AS `value_count` FROM `default`.`src` GROUP BY `src`.`key` -View Rewrite Enabled: No PREHOOK: query: SELECT * FROM view15 ORDER BY value_count DESC, key LIMIT 10 @@ -1510,11 +1501,10 @@ Bucket Columns: [] Sort Columns: [] # View Information -View Original Text: SELECT DISTINCT value +Original Query: SELECT DISTINCT value FROM src -View Expanded Text: SELECT DISTINCT `src`.`value` +Expanded Query: SELECT DISTINCT `src`.`value` FROM `default`.`src` -View Rewrite Enabled: No PREHOOK: query: SELECT * FROM view16 ORDER BY value LIMIT 10 diff --git a/ql/src/test/results/clientpositive/dbtxnmgr_query5.q.out b/ql/src/test/results/clientpositive/dbtxnmgr_query5.q.out index ea76a37512..37ff3ec649 100644 --- a/ql/src/test/results/clientpositive/dbtxnmgr_query5.q.out +++ b/ql/src/test/results/clientpositive/dbtxnmgr_query5.q.out @@ -43,7 +43,6 @@ POSTHOOK: query: show tables POSTHOOK: type: SHOWTABLES POSTHOOK: Input: database:foo t1_n40 -v1_n3 PREHOOK: query: describe T1_n40 PREHOOK: type: DESCTABLE PREHOOK: Input: foo@t1_n40 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out index 95f8966bea..36a3d8c3bf 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out @@ -62,10 +62,11 @@ Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -# View Information -View Original Text: select a, b, c from cmv_basetable_n4 -View Expanded Text: select `cmv_basetable_n4`.`a`, `cmv_basetable_n4`.`b`, `cmv_basetable_n4`.`c` from `default`.`cmv_basetable_n4` -View Rewrite Enabled: No +# Materialized View Information +Original Query: select a, b, c from cmv_basetable_n4 +Expanded Query: select `cmv_basetable_n4`.`a`, `cmv_basetable_n4`.`b`, `cmv_basetable_n4`.`c` from `default`.`cmv_basetable_n4` +Rewrite Enabled: No +Outdated for Rewriting: Unknown PREHOOK: query: select * from cmv_mat_view_n4 PREHOOK: type: QUERY PREHOOK: Input: default@cmv_mat_view_n4 @@ -123,10 +124,11 @@ Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -# View Information -View Original Text: select a, c from cmv_basetable_n4 -View Expanded Text: select `cmv_basetable_n4`.`a`, `cmv_basetable_n4`.`c` from `default`.`cmv_basetable_n4` -View Rewrite Enabled: No +# Materialized View Information +Original Query: select a, c from cmv_basetable_n4 +Expanded Query: select `cmv_basetable_n4`.`a`, `cmv_basetable_n4`.`c` from `default`.`cmv_basetable_n4` +Rewrite Enabled: No +Outdated for Rewriting: Unknown PREHOOK: query: select * from cmv_mat_view2_n1 PREHOOK: type: QUERY PREHOOK: Input: default@cmv_mat_view2_n1 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out index 42bfa4bbc9..fcd84665dd 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out @@ -284,16 +284,17 @@ Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -# View Information -View Original Text: SELECT cmv_basetable_n5.a, cmv_basetable_2_n2.c, sum(cmv_basetable_2_n2.d) +# Materialized View Information +Original Query: SELECT cmv_basetable_n5.a, cmv_basetable_2_n2.c, sum(cmv_basetable_2_n2.d) FROM cmv_basetable_n5 JOIN cmv_basetable_2_n2 ON (cmv_basetable_n5.a = cmv_basetable_2_n2.a) WHERE cmv_basetable_2_n2.c > 10.0 GROUP BY cmv_basetable_n5.a, cmv_basetable_2_n2.c -View Expanded Text: SELECT `cmv_basetable_n5`.`a`, `cmv_basetable_2_n2`.`c`, sum(`cmv_basetable_2_n2`.`d`) +Expanded Query: SELECT `cmv_basetable_n5`.`a`, `cmv_basetable_2_n2`.`c`, sum(`cmv_basetable_2_n2`.`d`) FROM `default`.`cmv_basetable_n5` JOIN `default`.`cmv_basetable_2_n2` ON (`cmv_basetable_n5`.`a` = `cmv_basetable_2_n2`.`a`) WHERE `cmv_basetable_2_n2`.`c` > 10.0 GROUP BY `cmv_basetable_n5`.`a`, `cmv_basetable_2_n2`.`c` -View Rewrite Enabled: No +Rewrite Enabled: No +Outdated for Rewriting: No PREHOOK: query: EXPLAIN SELECT cmv_basetable_n5.a, sum(cmv_basetable_2_n2.d) FROM cmv_basetable_n5 join cmv_basetable_2_n2 ON (cmv_basetable_n5.a = cmv_basetable_2_n2.a) @@ -513,16 +514,17 @@ Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -# View Information -View Original Text: SELECT cmv_basetable_n5.a, cmv_basetable_2_n2.c, sum(cmv_basetable_2_n2.d) +# Materialized View Information +Original Query: SELECT cmv_basetable_n5.a, cmv_basetable_2_n2.c, sum(cmv_basetable_2_n2.d) FROM cmv_basetable_n5 JOIN cmv_basetable_2_n2 ON (cmv_basetable_n5.a = cmv_basetable_2_n2.a) WHERE cmv_basetable_2_n2.c > 10.0 GROUP BY cmv_basetable_n5.a, cmv_basetable_2_n2.c -View Expanded Text: SELECT `cmv_basetable_n5`.`a`, `cmv_basetable_2_n2`.`c`, sum(`cmv_basetable_2_n2`.`d`) +Expanded Query: SELECT `cmv_basetable_n5`.`a`, `cmv_basetable_2_n2`.`c`, sum(`cmv_basetable_2_n2`.`d`) FROM `default`.`cmv_basetable_n5` JOIN `default`.`cmv_basetable_2_n2` ON (`cmv_basetable_n5`.`a` = `cmv_basetable_2_n2`.`a`) WHERE `cmv_basetable_2_n2`.`c` > 10.0 GROUP BY `cmv_basetable_n5`.`a`, `cmv_basetable_2_n2`.`c` -View Rewrite Enabled: Yes +Rewrite Enabled: Yes +Outdated for Rewriting: Yes PREHOOK: query: EXPLAIN SELECT cmv_basetable_n5.a, sum(cmv_basetable_2_n2.d) FROM cmv_basetable_n5 join cmv_basetable_2_n2 ON (cmv_basetable_n5.a = cmv_basetable_2_n2.a) @@ -964,16 +966,17 @@ Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -# View Information -View Original Text: SELECT cmv_basetable_n5.a, cmv_basetable_2_n2.c, sum(cmv_basetable_2_n2.d) +# Materialized View Information +Original Query: SELECT cmv_basetable_n5.a, cmv_basetable_2_n2.c, sum(cmv_basetable_2_n2.d) FROM cmv_basetable_n5 JOIN cmv_basetable_2_n2 ON (cmv_basetable_n5.a = cmv_basetable_2_n2.a) WHERE cmv_basetable_2_n2.c > 10.0 GROUP BY cmv_basetable_n5.a, cmv_basetable_2_n2.c -View Expanded Text: SELECT `cmv_basetable_n5`.`a`, `cmv_basetable_2_n2`.`c`, sum(`cmv_basetable_2_n2`.`d`) +Expanded Query: SELECT `cmv_basetable_n5`.`a`, `cmv_basetable_2_n2`.`c`, sum(`cmv_basetable_2_n2`.`d`) FROM `default`.`cmv_basetable_n5` JOIN `default`.`cmv_basetable_2_n2` ON (`cmv_basetable_n5`.`a` = `cmv_basetable_2_n2`.`a`) WHERE `cmv_basetable_2_n2`.`c` > 10.0 GROUP BY `cmv_basetable_n5`.`a`, `cmv_basetable_2_n2`.`c` -View Rewrite Enabled: Yes +Rewrite Enabled: Yes +Outdated for Rewriting: No PREHOOK: query: EXPLAIN SELECT cmv_basetable_n5.a, sum(cmv_basetable_2_n2.d) FROM cmv_basetable_n5 join cmv_basetable_2_n2 ON (cmv_basetable_n5.a = cmv_basetable_2_n2.a) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out index 949f71b3ca..5cb59acc0b 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out @@ -421,14 +421,15 @@ Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -# View Information -View Original Text: SELECT cmv_basetable_n6.a, cmv_basetable_2_n3.c +# Materialized View Information +Original Query: SELECT cmv_basetable_n6.a, cmv_basetable_2_n3.c FROM cmv_basetable_n6 JOIN cmv_basetable_2_n3 ON (cmv_basetable_n6.a = cmv_basetable_2_n3.a) WHERE cmv_basetable_2_n3.c > 10.0 -View Expanded Text: SELECT `cmv_basetable_n6`.`a`, `cmv_basetable_2_n3`.`c` +Expanded Query: SELECT `cmv_basetable_n6`.`a`, `cmv_basetable_2_n3`.`c` FROM `default`.`cmv_basetable_n6` JOIN `default`.`cmv_basetable_2_n3` ON (`cmv_basetable_n6`.`a` = `cmv_basetable_2_n3`.`a`) WHERE `cmv_basetable_2_n3`.`c` > 10.0 -View Rewrite Enabled: Yes +Rewrite Enabled: Yes +Outdated for Rewriting: No PREHOOK: query: EXPLAIN SELECT cmv_basetable_n6.a FROM cmv_basetable_n6 join cmv_basetable_2_n3 ON (cmv_basetable_n6.a = cmv_basetable_2_n3.a) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out index 5b957bd212..0601ea7f1a 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out @@ -269,16 +269,17 @@ Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -# View Information -View Original Text: SELECT cmv_basetable_n3.a, cmv_basetable_2_n1.c +# Materialized View Information +Original Query: SELECT cmv_basetable_n3.a, cmv_basetable_2_n1.c FROM cmv_basetable_n3 JOIN cmv_basetable_2_n1 ON (cmv_basetable_n3.a = cmv_basetable_2_n1.a) WHERE cmv_basetable_2_n1.c > 10.0 GROUP BY cmv_basetable_n3.a, cmv_basetable_2_n1.c -View Expanded Text: SELECT `cmv_basetable_n3`.`a`, `cmv_basetable_2_n1`.`c` +Expanded Query: SELECT `cmv_basetable_n3`.`a`, `cmv_basetable_2_n1`.`c` FROM `default`.`cmv_basetable_n3` JOIN `default`.`cmv_basetable_2_n1` ON (`cmv_basetable_n3`.`a` = `cmv_basetable_2_n1`.`a`) WHERE `cmv_basetable_2_n1`.`c` > 10.0 GROUP BY `cmv_basetable_n3`.`a`, `cmv_basetable_2_n1`.`c` -View Rewrite Enabled: No +Rewrite Enabled: No +Outdated for Rewriting: No PREHOOK: query: EXPLAIN SELECT cmv_basetable_n3.a FROM cmv_basetable_n3 join cmv_basetable_2_n1 ON (cmv_basetable_n3.a = cmv_basetable_2_n1.a) @@ -493,16 +494,17 @@ Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -# View Information -View Original Text: SELECT cmv_basetable_n3.a, cmv_basetable_2_n1.c +# Materialized View Information +Original Query: SELECT cmv_basetable_n3.a, cmv_basetable_2_n1.c FROM cmv_basetable_n3 JOIN cmv_basetable_2_n1 ON (cmv_basetable_n3.a = cmv_basetable_2_n1.a) WHERE cmv_basetable_2_n1.c > 10.0 GROUP BY cmv_basetable_n3.a, cmv_basetable_2_n1.c -View Expanded Text: SELECT `cmv_basetable_n3`.`a`, `cmv_basetable_2_n1`.`c` +Expanded Query: SELECT `cmv_basetable_n3`.`a`, `cmv_basetable_2_n1`.`c` FROM `default`.`cmv_basetable_n3` JOIN `default`.`cmv_basetable_2_n1` ON (`cmv_basetable_n3`.`a` = `cmv_basetable_2_n1`.`a`) WHERE `cmv_basetable_2_n1`.`c` > 10.0 GROUP BY `cmv_basetable_n3`.`a`, `cmv_basetable_2_n1`.`c` -View Rewrite Enabled: Yes +Rewrite Enabled: Yes +Outdated for Rewriting: No PREHOOK: query: EXPLAIN SELECT cmv_basetable_n3.a FROM cmv_basetable_n3 join cmv_basetable_2_n1 ON (cmv_basetable_n3.a = cmv_basetable_2_n1.a) @@ -799,16 +801,17 @@ Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -# View Information -View Original Text: SELECT cmv_basetable_n3.a, cmv_basetable_2_n1.c +# Materialized View Information +Original Query: SELECT cmv_basetable_n3.a, cmv_basetable_2_n1.c FROM cmv_basetable_n3 JOIN cmv_basetable_2_n1 ON (cmv_basetable_n3.a = cmv_basetable_2_n1.a) WHERE cmv_basetable_2_n1.c > 10.0 GROUP BY cmv_basetable_n3.a, cmv_basetable_2_n1.c -View Expanded Text: SELECT `cmv_basetable_n3`.`a`, `cmv_basetable_2_n1`.`c` +Expanded Query: SELECT `cmv_basetable_n3`.`a`, `cmv_basetable_2_n1`.`c` FROM `default`.`cmv_basetable_n3` JOIN `default`.`cmv_basetable_2_n1` ON (`cmv_basetable_n3`.`a` = `cmv_basetable_2_n1`.`a`) WHERE `cmv_basetable_2_n1`.`c` > 10.0 GROUP BY `cmv_basetable_n3`.`a`, `cmv_basetable_2_n1`.`c` -View Rewrite Enabled: Yes +Rewrite Enabled: Yes +Outdated for Rewriting: No PREHOOK: query: EXPLAIN SELECT cmv_basetable_n3.a FROM cmv_basetable_n3 join cmv_basetable_2_n1 ON (cmv_basetable_n3.a = cmv_basetable_2_n1.a) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window_2.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window_2.q.out new file mode 100644 index 0000000000..24bd61d7f8 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window_2.q.out @@ -0,0 +1,291 @@ +PREHOOK: query: create table cmv_basetable_n100 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable_n100 +POSTHOOK: query: create table cmv_basetable_n100 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable_n100 +PREHOOK: query: insert into cmv_basetable_n100 values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable_n100 +POSTHOOK: query: insert into cmv_basetable_n100 values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable_n100 +POSTHOOK: Lineage: cmv_basetable_n100.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_n100.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_n100.c SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_n100.d SCRIPT [] +PREHOOK: query: analyze table cmv_basetable_n100 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@cmv_basetable_n100 +PREHOOK: Output: default@cmv_basetable_n100 +#### A masked pattern was here #### +POSTHOOK: query: analyze table cmv_basetable_n100 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@cmv_basetable_n100 +POSTHOOK: Output: default@cmv_basetable_n100 +#### A masked pattern was here #### +PREHOOK: query: create table cmv_basetable_2_n100 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable_2_n100 +POSTHOOK: query: create table cmv_basetable_2_n100 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable_2_n100 +PREHOOK: query: insert into cmv_basetable_2_n100 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable_2_n100 +POSTHOOK: query: insert into cmv_basetable_2_n100 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable_2_n100 +POSTHOOK: Lineage: cmv_basetable_2_n100.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2_n100.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2_n100.c SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2_n100.d SCRIPT [] +PREHOOK: query: analyze table cmv_basetable_2_n100 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@cmv_basetable_2_n100 +PREHOOK: Output: default@cmv_basetable_2_n100 +#### A masked pattern was here #### +POSTHOOK: query: analyze table cmv_basetable_2_n100 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@cmv_basetable_2_n100 +POSTHOOK: Output: default@cmv_basetable_2_n100 +#### A masked pattern was here #### +PREHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view_n300 AS + SELECT cmv_basetable_n100.a, cmv_basetable_2_n100.c + FROM cmv_basetable_n100 JOIN cmv_basetable_2_n100 ON (cmv_basetable_n100.a = cmv_basetable_2_n100.a) + WHERE cmv_basetable_2_n100.c > 10.0 + GROUP BY cmv_basetable_n100.a, cmv_basetable_2_n100.c +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable_2_n100 +PREHOOK: Input: default@cmv_basetable_n100 +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view_n300 +POSTHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view_n300 AS + SELECT cmv_basetable_n100.a, cmv_basetable_2_n100.c + FROM cmv_basetable_n100 JOIN cmv_basetable_2_n100 ON (cmv_basetable_n100.a = cmv_basetable_2_n100.a) + WHERE cmv_basetable_2_n100.c > 10.0 + GROUP BY cmv_basetable_n100.a, cmv_basetable_2_n100.c +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable_2_n100 +POSTHOOK: Input: default@cmv_basetable_n100 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view_n300 +PREHOOK: query: DESCRIBE FORMATTED cmv_mat_view_n300 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cmv_mat_view_n300 +POSTHOOK: query: DESCRIBE FORMATTED cmv_mat_view_n300 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cmv_mat_view_n300 +# col_name data_type comment +a int +c decimal(10,2) + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + numFiles 2 + numRows 2 + rawDataSize 232 + totalSize 608 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: SELECT cmv_basetable_n100.a, cmv_basetable_2_n100.c + FROM cmv_basetable_n100 JOIN cmv_basetable_2_n100 ON (cmv_basetable_n100.a = cmv_basetable_2_n100.a) + WHERE cmv_basetable_2_n100.c > 10.0 + GROUP BY cmv_basetable_n100.a, cmv_basetable_2_n100.c +Expanded Query: SELECT `cmv_basetable_n100`.`a`, `cmv_basetable_2_n100`.`c` + FROM `default`.`cmv_basetable_n100` JOIN `default`.`cmv_basetable_2_n100` ON (`cmv_basetable_n100`.`a` = `cmv_basetable_2_n100`.`a`) + WHERE `cmv_basetable_2_n100`.`c` > 10.0 + GROUP BY `cmv_basetable_n100`.`a`, `cmv_basetable_2_n100`.`c` +Rewrite Enabled: Yes +Outdated for Rewriting: No +PREHOOK: query: insert into cmv_basetable_2_n100 values + (3, 'charlie', 15.8, 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable_2_n100 +POSTHOOK: query: insert into cmv_basetable_2_n100 values + (3, 'charlie', 15.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable_2_n100 +POSTHOOK: Lineage: cmv_basetable_2_n100.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2_n100.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2_n100.c SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2_n100.d SCRIPT [] +PREHOOK: query: analyze table cmv_basetable_2_n100 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@cmv_basetable_2_n100 +PREHOOK: Output: default@cmv_basetable_2_n100 +#### A masked pattern was here #### +POSTHOOK: query: analyze table cmv_basetable_2_n100 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@cmv_basetable_2_n100 +POSTHOOK: Output: default@cmv_basetable_2_n100 +#### A masked pattern was here #### +PREHOOK: query: DESCRIBE FORMATTED cmv_mat_view_n300 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cmv_mat_view_n300 +POSTHOOK: query: DESCRIBE FORMATTED cmv_mat_view_n300 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cmv_mat_view_n300 +# col_name data_type comment +a int +c decimal(10,2) + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + numFiles 2 + numRows 2 + rawDataSize 232 + totalSize 608 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: SELECT cmv_basetable_n100.a, cmv_basetable_2_n100.c + FROM cmv_basetable_n100 JOIN cmv_basetable_2_n100 ON (cmv_basetable_n100.a = cmv_basetable_2_n100.a) + WHERE cmv_basetable_2_n100.c > 10.0 + GROUP BY cmv_basetable_n100.a, cmv_basetable_2_n100.c +Expanded Query: SELECT `cmv_basetable_n100`.`a`, `cmv_basetable_2_n100`.`c` + FROM `default`.`cmv_basetable_n100` JOIN `default`.`cmv_basetable_2_n100` ON (`cmv_basetable_n100`.`a` = `cmv_basetable_2_n100`.`a`) + WHERE `cmv_basetable_2_n100`.`c` > 10.0 + GROUP BY `cmv_basetable_n100`.`a`, `cmv_basetable_2_n100`.`c` +Rewrite Enabled: Yes +Outdated for Rewriting: Yes +PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view_n300 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable_2_n100 +PREHOOK: Input: default@cmv_basetable_n100 +PREHOOK: Input: default@cmv_mat_view_n300 +PREHOOK: Output: default@cmv_mat_view_n300 +POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view_n300 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable_2_n100 +POSTHOOK: Input: default@cmv_basetable_n100 +POSTHOOK: Input: default@cmv_mat_view_n300 +POSTHOOK: Output: default@cmv_mat_view_n300 +POSTHOOK: Lineage: cmv_mat_view_n300.a EXPRESSION [(cmv_basetable_n100)cmv_basetable_n100.FieldSchema(name:a, type:int, comment:null), (cmv_mat_view_n300)default.cmv_mat_view_n300.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view_n300.c EXPRESSION [(cmv_basetable_2_n100)cmv_basetable_2_n100.FieldSchema(name:c, type:decimal(10,2), comment:null), (cmv_mat_view_n300)default.cmv_mat_view_n300.FieldSchema(name:c, type:decimal(10,2), comment:null), ] +PREHOOK: query: DESCRIBE FORMATTED cmv_mat_view_n300 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cmv_mat_view_n300 +POSTHOOK: query: DESCRIBE FORMATTED cmv_mat_view_n300 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cmv_mat_view_n300 +# col_name data_type comment +a int +c decimal(10,2) + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"c\":\"true\"}} + bucketing_version 2 + numFiles 2 + numRows 3 + rawDataSize 348 + totalSize 628 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: SELECT cmv_basetable_n100.a, cmv_basetable_2_n100.c + FROM cmv_basetable_n100 JOIN cmv_basetable_2_n100 ON (cmv_basetable_n100.a = cmv_basetable_2_n100.a) + WHERE cmv_basetable_2_n100.c > 10.0 + GROUP BY cmv_basetable_n100.a, cmv_basetable_2_n100.c +Expanded Query: SELECT `cmv_basetable_n100`.`a`, `cmv_basetable_2_n100`.`c` + FROM `default`.`cmv_basetable_n100` JOIN `default`.`cmv_basetable_2_n100` ON (`cmv_basetable_n100`.`a` = `cmv_basetable_2_n100`.`a`) + WHERE `cmv_basetable_2_n100`.`c` > 10.0 + GROUP BY `cmv_basetable_n100`.`a`, `cmv_basetable_2_n100`.`c` +Rewrite Enabled: Yes +Outdated for Rewriting: No +PREHOOK: query: drop materialized view cmv_mat_view_n300 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view_n300 +PREHOOK: Output: default@cmv_mat_view_n300 +POSTHOOK: query: drop materialized view cmv_mat_view_n300 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view_n300 +POSTHOOK: Output: default@cmv_mat_view_n300 +PREHOOK: query: drop table cmv_basetable_n100 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@cmv_basetable_n100 +PREHOOK: Output: default@cmv_basetable_n100 +POSTHOOK: query: drop table cmv_basetable_n100 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@cmv_basetable_n100 +POSTHOOK: Output: default@cmv_basetable_n100 +PREHOOK: query: drop table cmv_basetable_2_n100 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@cmv_basetable_2_n100 +PREHOOK: Output: default@cmv_basetable_2_n100 +POSTHOOK: query: drop table cmv_basetable_2_n100 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@cmv_basetable_2_n100 +POSTHOOK: Output: default@cmv_basetable_2_n100 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out index c68c127a02..2928fcfb9b 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out @@ -85,10 +85,11 @@ Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -# View Information -View Original Text: select a, c from cmv_basetable_n8 -View Expanded Text: select `cmv_basetable_n8`.`a`, `cmv_basetable_n8`.`c` from `default`.`cmv_basetable_n8` -View Rewrite Enabled: No +# Materialized View Information +Original Query: select a, c from cmv_basetable_n8 +Expanded Query: select `cmv_basetable_n8`.`a`, `cmv_basetable_n8`.`c` from `default`.`cmv_basetable_n8` +Rewrite Enabled: No +Outdated for Rewriting: Unknown PREHOOK: query: show tblproperties cmv_mat_view_n8 PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view_n8 @@ -176,10 +177,11 @@ Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -# View Information -View Original Text: select a from cmv_basetable_n8 -View Expanded Text: select `cmv_basetable_n8`.`a` from `default`.`cmv_basetable_n8` -View Rewrite Enabled: No +# Materialized View Information +Original Query: select a from cmv_basetable_n8 +Expanded Query: select `cmv_basetable_n8`.`a` from `default`.`cmv_basetable_n8` +Rewrite Enabled: No +Outdated for Rewriting: Unknown PREHOOK: query: select a from cmv_mat_view2_n3 PREHOOK: type: QUERY PREHOOK: Input: default@cmv_mat_view2_n3 @@ -255,10 +257,11 @@ Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -# View Information -View Original Text: select * from cmv_basetable_n8 -View Expanded Text: select `cmv_basetable_n8`.`a`, `cmv_basetable_n8`.`b`, `cmv_basetable_n8`.`c` from `default`.`cmv_basetable_n8` -View Rewrite Enabled: No +# Materialized View Information +Original Query: select * from cmv_basetable_n8 +Expanded Query: select `cmv_basetable_n8`.`a`, `cmv_basetable_n8`.`b`, `cmv_basetable_n8`.`c` from `default`.`cmv_basetable_n8` +Rewrite Enabled: No +Outdated for Rewriting: Unknown PREHOOK: query: select a, b, c from cmv_mat_view3_n0 PREHOOK: type: QUERY PREHOOK: Input: default@cmv_mat_view3_n0 @@ -345,10 +348,11 @@ Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -# View Information -View Original Text: select a from cmv_basetable_n8 -View Expanded Text: select `cmv_basetable_n8`.`a` from `default`.`cmv_basetable_n8` -View Rewrite Enabled: No +# Materialized View Information +Original Query: select a from cmv_basetable_n8 +Expanded Query: select `cmv_basetable_n8`.`a` from `default`.`cmv_basetable_n8` +Rewrite Enabled: No +Outdated for Rewriting: Unknown PREHOOK: query: select a from cmv_mat_view4_n0 PREHOOK: type: QUERY PREHOOK: Input: default@cmv_mat_view4_n0 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out index 3852945776..b12df11a98 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out @@ -204,10 +204,11 @@ Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -# View Information -View Original Text: SELECT value, key FROM src_txn where key > 200 and key < 250 -View Expanded Text: SELECT `src_txn`.`value`, `src_txn`.`key` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250 -View Rewrite Enabled: Yes +# Materialized View Information +Original Query: SELECT value, key FROM src_txn where key > 200 and key < 250 +Expanded Query: SELECT `src_txn`.`value`, `src_txn`.`key` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250 +Rewrite Enabled: Yes +Outdated for Rewriting: No PREHOOK: query: EXPLAIN SELECT * FROM partition_mv_1 where key = 238 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/temp_table.q.out b/ql/src/test/results/clientpositive/llap/temp_table.q.out index 2965d54974..80972b4dfc 100644 --- a/ql/src/test/results/clientpositive/llap/temp_table.q.out +++ b/ql/src/test/results/clientpositive/llap/temp_table.q.out @@ -154,14 +154,40 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@foo key string value string -PREHOOK: query: DESCRIBE bar +PREHOOK: query: DESCRIBE FORMATTED bar PREHOOK: type: DESCTABLE PREHOOK: Input: default@bar -POSTHOOK: query: DESCRIBE bar +POSTHOOK: query: DESCRIBE FORMATTED bar POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@bar +# col_name data_type comment key string value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + numFiles 1 + numRows 253 + rawDataSize 2703 + totalSize 2956 + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: explain select * from foo order by key limit 10 PREHOOK: type: QUERY POSTHOOK: query: explain select * from foo order by key limit 10 diff --git a/ql/src/test/results/clientpositive/show_materialized_views.q.out b/ql/src/test/results/clientpositive/show_materialized_views.q.out index 715afea92c..2a00902ee0 100644 --- a/ql/src/test/results/clientpositive/show_materialized_views.q.out +++ b/ql/src/test/results/clientpositive/show_materialized_views.q.out @@ -38,25 +38,29 @@ POSTHOOK: type: CREATE_MATERIALIZED_VIEW POSTHOOK: Input: test1@shtb_test1 POSTHOOK: Output: database:test1 POSTHOOK: Output: test1@shtb_test1_view1 -PREHOOK: query: CREATE MATERIALIZED VIEW shtb_test1_view2 AS +PREHOOK: query: CREATE MATERIALIZED VIEW shtb_test1_view2 +TBLPROPERTIES ('rewriting.time.window' = '-1min') AS SELECT * FROM shtb_test1 where KEY > 100 and KEY < 200 PREHOOK: type: CREATE_MATERIALIZED_VIEW PREHOOK: Input: test1@shtb_test1 PREHOOK: Output: database:test1 PREHOOK: Output: test1@shtb_test1_view2 -POSTHOOK: query: CREATE MATERIALIZED VIEW shtb_test1_view2 AS +POSTHOOK: query: CREATE MATERIALIZED VIEW shtb_test1_view2 +TBLPROPERTIES ('rewriting.time.window' = '-1min') AS SELECT * FROM shtb_test1 where KEY > 100 and KEY < 200 POSTHOOK: type: CREATE_MATERIALIZED_VIEW POSTHOOK: Input: test1@shtb_test1 POSTHOOK: Output: database:test1 POSTHOOK: Output: test1@shtb_test1_view2 -PREHOOK: query: CREATE MATERIALIZED VIEW shtb_full_view2 AS +PREHOOK: query: CREATE MATERIALIZED VIEW shtb_full_view2 +TBLPROPERTIES ('rewriting.time.window' = '5min') AS SELECT * FROM shtb_test1 PREHOOK: type: CREATE_MATERIALIZED_VIEW PREHOOK: Input: test1@shtb_test1 PREHOOK: Output: database:test1 PREHOOK: Output: test1@shtb_full_view2 -POSTHOOK: query: CREATE MATERIALIZED VIEW shtb_full_view2 AS +POSTHOOK: query: CREATE MATERIALIZED VIEW shtb_full_view2 +TBLPROPERTIES ('rewriting.time.window' = '5min') AS SELECT * FROM shtb_test1 POSTHOOK: type: CREATE_MATERIALIZED_VIEW POSTHOOK: Input: test1@shtb_test1 @@ -122,23 +126,38 @@ PREHOOK: query: SHOW MATERIALIZED VIEWS PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS POSTHOOK: type: SHOWMATERIALIZEDVIEWS -shtb_full_view2 -shtb_test1_view1 -shtb_test1_view2 +# MV Name Rewriting Enabled Mode +shtb_full_view2 Yes Manual refresh (Valid for 5min) +shtb_test1_view1 No Manual refresh +shtb_test1_view2 Yes Manual refresh (Valid always) + PREHOOK: query: SHOW MATERIALIZED VIEWS 'test_*' PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS 'test_*' POSTHOOK: type: SHOWMATERIALIZEDVIEWS +# MV Name Rewriting Enabled Mode +shtb_full_view2 Yes Manual refresh (Valid for 5min) +shtb_test1_view1 No Manual refresh +shtb_test1_view2 Yes Manual refresh (Valid always) + PREHOOK: query: SHOW MATERIALIZED VIEWS '*view2' PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS '*view2' POSTHOOK: type: SHOWMATERIALIZEDVIEWS -shtb_full_view2 -shtb_test1_view2 +# MV Name Rewriting Enabled Mode +shtb_full_view2 Yes Manual refresh (Valid for 5min) +shtb_test1_view1 No Manual refresh +shtb_test1_view2 Yes Manual refresh (Valid always) + PREHOOK: query: SHOW MATERIALIZED VIEWS LIKE 'test_view1|test_view2' PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS LIKE 'test_view1|test_view2' POSTHOOK: type: SHOWMATERIALIZEDVIEWS +# MV Name Rewriting Enabled Mode +shtb_full_view2 Yes Manual refresh (Valid for 5min) +shtb_test1_view1 No Manual refresh +shtb_test1_view2 Yes Manual refresh (Valid always) + PREHOOK: query: USE test2 PREHOOK: type: SWITCHDATABASE PREHOOK: Input: database:test2 @@ -149,8 +168,10 @@ PREHOOK: query: SHOW MATERIALIZED VIEWS 'shtb_*' PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS 'shtb_*' POSTHOOK: type: SHOWMATERIALIZEDVIEWS -shtb_test1_view1 -shtb_test2_view2 +# MV Name Rewriting Enabled Mode +shtb_test1_view1 No Manual refresh +shtb_test2_view2 No Manual refresh + PREHOOK: query: USE default PREHOOK: type: SWITCHDATABASE PREHOOK: Input: database:default @@ -161,22 +182,28 @@ PREHOOK: query: SHOW MATERIALIZED VIEWS FROM test1 PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS FROM test1 POSTHOOK: type: SHOWMATERIALIZEDVIEWS -shtb_full_view2 -shtb_test1_view1 -shtb_test1_view2 +# MV Name Rewriting Enabled Mode +shtb_full_view2 Yes Manual refresh (Valid for 5min) +shtb_test1_view1 No Manual refresh +shtb_test1_view2 Yes Manual refresh (Valid always) + PREHOOK: query: SHOW MATERIALIZED VIEWS FROM test2 PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS FROM test2 POSTHOOK: type: SHOWMATERIALIZEDVIEWS -shtb_test1_view1 -shtb_test2_view2 +# MV Name Rewriting Enabled Mode +shtb_test1_view1 No Manual refresh +shtb_test2_view2 No Manual refresh + PREHOOK: query: SHOW MATERIALIZED VIEWS IN test1 PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS IN test1 POSTHOOK: type: SHOWMATERIALIZEDVIEWS -shtb_full_view2 -shtb_test1_view1 -shtb_test1_view2 +# MV Name Rewriting Enabled Mode +shtb_full_view2 Yes Manual refresh (Valid for 5min) +shtb_test1_view1 No Manual refresh +shtb_test1_view2 Yes Manual refresh (Valid always) + PREHOOK: query: SHOW MATERIALIZED VIEWS IN default PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS IN default @@ -185,10 +212,141 @@ PREHOOK: query: SHOW MATERIALIZED VIEWS IN test1 "shtb_test_*" PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS IN test1 "shtb_test_*" POSTHOOK: type: SHOWMATERIALIZEDVIEWS +# MV Name Rewriting Enabled Mode +shtb_full_view2 Yes Manual refresh (Valid for 5min) +shtb_test1_view1 No Manual refresh +shtb_test1_view2 Yes Manual refresh (Valid always) + +PREHOOK: query: DESCRIBE FORMATTED test1.shtb_full_view2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: test1@shtb_full_view2 +POSTHOOK: query: DESCRIBE FORMATTED test1.shtb_full_view2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: test1@shtb_full_view2 +# col_name data_type comment +key int +value string +ds string + +# Detailed Table Information +Database: test1 +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + numFiles 0 + numRows 0 + rawDataSize 0 + rewriting.time.window 5min + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: SELECT * FROM shtb_test1 +Expanded Query: SELECT `shtb_test1`.`key`, `shtb_test1`.`value`, `shtb_test1`.`ds` FROM `test1`.`shtb_test1` +Rewrite Enabled: Yes +Outdated for Rewriting: No +PREHOOK: query: DESCRIBE FORMATTED test1.shtb_test1_view1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: test1@shtb_test1_view1 +POSTHOOK: query: DESCRIBE FORMATTED test1.shtb_test1_view1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: test1@shtb_test1_view1 +# col_name data_type comment +key int +value string +ds string + +# Detailed Table Information +Database: test1 +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: SELECT * FROM shtb_test1 where KEY > 1000 and KEY < 2000 +Expanded Query: SELECT `shtb_test1`.`key`, `shtb_test1`.`value`, `shtb_test1`.`ds` FROM `test1`.`shtb_test1` where `shtb_test1`.`key` > 1000 and `shtb_test1`.`key` < 2000 +Rewrite Enabled: No +Outdated for Rewriting: No +PREHOOK: query: DESCRIBE FORMATTED test1.shtb_test1_view2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: test1@shtb_test1_view2 +POSTHOOK: query: DESCRIBE FORMATTED test1.shtb_test1_view2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: test1@shtb_test1_view2 +# col_name data_type comment +key int +value string +ds string + +# Detailed Table Information +Database: test1 +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + numFiles 0 + numRows 0 + rawDataSize 0 + rewriting.time.window -1min + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: SELECT * FROM shtb_test1 where KEY > 100 and KEY < 200 +Expanded Query: SELECT `shtb_test1`.`key`, `shtb_test1`.`value`, `shtb_test1`.`ds` FROM `test1`.`shtb_test1` where `shtb_test1`.`key` > 100 and `shtb_test1`.`key` < 200 +Rewrite Enabled: Yes +Outdated for Rewriting: No PREHOOK: query: SHOW MATERIALIZED VIEWS IN test2 LIKE "nomatch" PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS IN test2 LIKE "nomatch" POSTHOOK: type: SHOWMATERIALIZEDVIEWS +# MV Name Rewriting Enabled Mode +shtb_test1_view1 No Manual refresh +shtb_test2_view2 No Manual refresh + PREHOOK: query: CREATE DATABASE `database` PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:database @@ -202,26 +360,29 @@ POSTHOOK: query: USE `database` POSTHOOK: type: SWITCHDATABASE POSTHOOK: Input: database:database PREHOOK: query: CREATE TABLE foo_n0(a INT) +STORED AS ORC TBLPROPERTIES ('transactional'='true') PREHOOK: type: CREATETABLE PREHOOK: Output: database:database PREHOOK: Output: database@foo_n0 POSTHOOK: query: CREATE TABLE foo_n0(a INT) +STORED AS ORC TBLPROPERTIES ('transactional'='true') POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:database POSTHOOK: Output: database@foo_n0 -PREHOOK: query: CREATE VIEW fooview AS +PREHOOK: query: CREATE MATERIALIZED VIEW fooview +TBLPROPERTIES ('rewriting.time.window' = '0min') AS SELECT * FROM foo_n0 -PREHOOK: type: CREATEVIEW +PREHOOK: type: CREATE_MATERIALIZED_VIEW PREHOOK: Input: database@foo_n0 PREHOOK: Output: database:database PREHOOK: Output: database@fooview -POSTHOOK: query: CREATE VIEW fooview AS +POSTHOOK: query: CREATE MATERIALIZED VIEW fooview +TBLPROPERTIES ('rewriting.time.window' = '0min') AS SELECT * FROM foo_n0 -POSTHOOK: type: CREATEVIEW +POSTHOOK: type: CREATE_MATERIALIZED_VIEW POSTHOOK: Input: database@foo_n0 POSTHOOK: Output: database:database POSTHOOK: Output: database@fooview -POSTHOOK: Lineage: fooview.a SIMPLE [(foo_n0)foo_n0.FieldSchema(name:a, type:int, comment:null), ] PREHOOK: query: USE default PREHOOK: type: SWITCHDATABASE PREHOOK: Input: database:default @@ -232,6 +393,48 @@ PREHOOK: query: SHOW MATERIALIZED VIEWS FROM `database` LIKE "fooview" PREHOOK: type: SHOWMATERIALIZEDVIEWS POSTHOOK: query: SHOW MATERIALIZED VIEWS FROM `database` LIKE "fooview" POSTHOOK: type: SHOWMATERIALIZEDVIEWS +# MV Name Rewriting Enabled Mode +fooview Yes Manual refresh (Valid until source tables modified) + +PREHOOK: query: DESCRIBE FORMATTED `database`.`fooview` +PREHOOK: type: DESCTABLE +PREHOOK: Input: database@fooview +POSTHOOK: query: DESCRIBE FORMATTED `database`.`fooview` +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: database@fooview +# col_name data_type comment +a int + +# Detailed Table Information +Database: database +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + numFiles 0 + numRows 0 + rawDataSize 0 + rewriting.time.window 0min + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: SELECT * FROM foo_n0 +Expanded Query: SELECT `foo_n0`.`a` FROM `database`.`foo_n0` +Rewrite Enabled: Yes +Outdated for Rewriting: No PREHOOK: query: DROP MATERIALIZED VIEW fooview PREHOOK: type: DROP_MATERIALIZED_VIEW POSTHOOK: query: DROP MATERIALIZED VIEW fooview diff --git a/ql/src/test/results/clientpositive/spark/temp_table.q.out b/ql/src/test/results/clientpositive/spark/temp_table.q.out index dabcbce635..b1f2ee1d41 100644 --- a/ql/src/test/results/clientpositive/spark/temp_table.q.out +++ b/ql/src/test/results/clientpositive/spark/temp_table.q.out @@ -144,14 +144,40 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@foo key string value string -PREHOOK: query: DESCRIBE bar +PREHOOK: query: DESCRIBE FORMATTED bar PREHOOK: type: DESCTABLE PREHOOK: Input: default@bar -POSTHOOK: query: DESCRIBE bar +POSTHOOK: query: DESCRIBE FORMATTED bar POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@bar +# col_name data_type comment key string value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + numFiles 1 + numRows 253 + rawDataSize 2703 + totalSize 2956 + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: explain select * from foo order by key limit 10 PREHOOK: type: QUERY POSTHOOK: query: explain select * from foo order by key limit 10