diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 0627c35378..40d05b954a 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1594,6 +1594,16 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "moment in time t0, the materialized view will not be considered for rewriting anymore after t0 plus " + "the value assigned to this property. Default value 0 means that the materialized view cannot be " + "outdated to be used automatically in query rewriting."), + HIVE_MATERIALIZED_VIEW_REWRITING_INCREMENTAL("hive.materializedview.rewriting.incremental", true, + "Whether to try to execute incremental rewritings based on outdated materializations and\n" + + "current content of tables. Default value of true effectively amounts to enabling incremental\n" + + "rebuild for the materializations too."), + HIVE_MATERIALIZED_VIEW_REBUILD_INCREMENTAL("hive.materializedview.rebuild.incremental", true, + "Whether to try to execute incremental rebuild for the materialized views. Incremental rebuild\n" + + "tries to modify the original materialization contents to reflect the latest changes to the\n" + + "materialized view source tables, instead of rebuilding the contents fully. Incremental rebuild\n" + + "is based on the materialized view algebraic incremental rewriting. Hence, this requires\n" + + "hive.materializedview.rewriting.incremental to be true."), HIVE_MATERIALIZED_VIEW_FILE_FORMAT("hive.materializedview.fileformat", "ORC", new StringSet("none", "TextFile", "SequenceFile", "RCfile", "ORC"), "Default file format for CREATE MATERIALIZED VIEW statement"), diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 28c14ebc4c..48d62a8bf9 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -206,15 +206,6 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ load_dyn_part2.q,\ load_dyn_part3.q,\ lvj_mapjoin.q,\ - materialized_view_create_rewrite.q,\ - materialized_view_create_rewrite_3.q,\ - materialized_view_describe.q,\ - materialized_view_rewrite_ssb.q,\ - materialized_view_create.q,\ - materialized_view_create_rewrite_2.q,\ - materialized_view_create_rewrite_multi_db.q,\ - materialized_view_drop.q,\ - materialized_view_rewrite_ssb_2.q,\ mapjoin2.q,\ mapjoin3.q,\ mapjoin_decimal.q,\ @@ -594,8 +585,29 @@ minillaplocal.query.files=\ load_data_acid_rename.q,\ load_dyn_part5.q,\ lvj_mapjoin.q,\ + materialized_view_create.q,\ materialized_view_create_rewrite_dummy.q,\ + materialized_view_create_rewrite_multi_db.q,\ materialized_view_create_rewrite_rebuild_dummy.q,\ + materialized_view_create_rewrite_time_window.q,\ + materialized_view_create_rewrite.q,\ + materialized_view_create_rewrite_2.q,\ + materialized_view_create_rewrite_3.q,\ + materialized_view_create_rewrite_4.q,\ + materialized_view_create_rewrite_5.q,\ + materialized_view_describe.q,\ + materialized_view_drop.q,\ + materialized_view_rewrite_1.q,\ + materialized_view_rewrite_2.q,\ + materialized_view_rewrite_3.q,\ + materialized_view_rewrite_4.q,\ + materialized_view_rewrite_5.q,\ + materialized_view_rewrite_6.q,\ + materialized_view_rewrite_7.q,\ + materialized_view_rewrite_8.q,\ + materialized_view_rewrite_9.q,\ + materialized_view_rewrite_ssb.q,\ + materialized_view_rewrite_ssb_2.q,\ mapjoin_decimal.q,\ mapjoin_hint.q,\ mapjoin_emit_interval.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index fb1efe01dc..bda2af3a04 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -64,6 +64,7 @@ import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.ValidTxnList; +import org.apache.hadoop.hive.common.ValidTxnWriteIdList; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.conf.Constants; @@ -5057,7 +5058,7 @@ private int createView(Hive db, CreateViewDesc crtView) throws HiveException { CreationMetadata cm = new CreationMetadata(MetaStoreUtils.getDefaultCatalog(conf), tbl.getDbName(), tbl.getTableName(), ImmutableSet.copyOf(crtView.getTablesUsed())); - cm.setValidTxnList(conf.get(ValidTxnList.VALID_TXNS_KEY)); + cm.setValidTxnList(conf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY)); tbl.getTTable().setCreationMetadata(cm); } db.createTable(tbl, crtView.getIfNotExists()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java index 50fc4e0c63..834df84869 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java @@ -20,6 +20,8 @@ import com.google.common.collect.ImmutableSet; import org.apache.hadoop.hive.common.ValidTxnList; +import org.apache.hadoop.hive.common.ValidTxnWriteIdList; +import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.metastore.api.CreationMetadata; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.DriverContext; @@ -69,7 +71,7 @@ public int execute(DriverContext driverContext) { new CreationMetadata(MetaStoreUtils.getDefaultCatalog(conf), mvTable.getDbName(), mvTable.getTableName(), ImmutableSet.copyOf(mvTable.getCreationMetadata().getTablesUsed())); - cm.setValidTxnList(conf.get(ValidTxnList.VALID_TXNS_KEY)); + cm.setValidTxnList(conf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY)); db.updateCreationMetadata(mvTable.getDbName(), mvTable.getTableName(), cm); } } catch (HiveException e) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java index 5a95649f5b..f566842e19 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java @@ -61,6 +61,7 @@ Licensed to the Apache Software Foundation (ASF) under one import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; /** * An implementation of HiveTxnManager that stores the transactions in the metastore database. @@ -754,6 +755,13 @@ private Heartbeater startHeartbeat(long initialDelay) throws LockException { } Heartbeater heartbeater = new Heartbeater(this, conf, queryId, currentUser); + heartbeatTask = startHeartbeat(initialDelay, heartbeatInterval, heartbeater); + LOG.debug("Started heartbeat with delay/interval = " + initialDelay + "/" + heartbeatInterval + + " " + TimeUnit.MILLISECONDS + " for query: " + queryId); + return heartbeater; + } + + private ScheduledFuture startHeartbeat(long initialDelay, long heartbeatInterval, Runnable heartbeater) { // For negative testing purpose.. if(conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST) && conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILHEARTBEATER)) { initialDelay = 0; @@ -763,12 +771,9 @@ private Heartbeater startHeartbeat(long initialDelay) throws LockException { resources, that they all don't start heartbeating at the same time*/ initialDelay = (long)Math.floor(heartbeatInterval * 0.75 * Math.random()); } - - heartbeatTask = heartbeatExecutorService.scheduleAtFixedRate( + ScheduledFuture task = heartbeatExecutorService.scheduleAtFixedRate( heartbeater, initialDelay, heartbeatInterval, TimeUnit.MILLISECONDS); - LOG.info("Started heartbeat with delay/interval = " + initialDelay + "/" + heartbeatInterval + - " " + TimeUnit.MILLISECONDS + " for query: " + queryId); - return heartbeater; + return task; } private void stopHeartbeat() throws LockException { @@ -886,6 +891,7 @@ else if(!isExplicitTransaction) { } return false; } + @Override public boolean isImplicitTransactionOpen() { if(!isTxnOpen()) { @@ -976,6 +982,38 @@ public long getTableWriteId(String dbName, String tableName) throws LockExceptio } } + @Override + public LockResponse acquireMaterializationRebuildLock(String dbName, String tableName, long txnId) throws LockException { + // Acquire lock + LockResponse lockResponse; + try { + lockResponse = getMS().lockMaterializationRebuild(dbName, tableName, txnId); + } catch (TException e) { + throw new LockException(ErrorMsg.METASTORE_COMMUNICATION_FAILED.getMsg(), e); + } + if (lockResponse.getState() == LockState.ACQUIRED) { + // If lock response is ACQUIRED, we can create the heartbeater + long initialDelay = 0L; + long heartbeatInterval = getHeartbeatInterval(conf); + assert heartbeatInterval > 0; + MaterializationRebuildLockHeartbeater heartbeater = new MaterializationRebuildLockHeartbeater( + this, dbName, tableName, queryId, txnId); + ScheduledFuture task = startHeartbeat(initialDelay, heartbeatInterval, heartbeater); + heartbeater.task.set(task); + LOG.debug("Started heartbeat for materialization rebuild lock for {} with delay/interval = {}/{} {} for query: {}", + AcidUtils.getFullTableName(dbName, tableName), initialDelay, heartbeatInterval, TimeUnit.MILLISECONDS, queryId); + } + return lockResponse; + } + + private boolean heartbeatMaterializationRebuildLock(String dbName, String tableName, long txnId) throws LockException { + try { + return getMS().heartbeatLockMaterializationRebuild(dbName, tableName, txnId); + } catch (TException e) { + throw new LockException(ErrorMsg.METASTORE_COMMUNICATION_FAILED.getMsg(), e); + } + } + private static long getHeartbeatInterval(Configuration conf) throws LockException { // Retrieve HIVE_TXN_TIMEOUT in MILLISECONDS (it's defined as SECONDS), // then divide it by 2 to give us a safety factor. @@ -1042,4 +1080,55 @@ public void run() { } } } + + /** + * MaterializationRebuildLockHeartbeater is a runnable that will be run in a + * ScheduledExecutorService in given intervals. Once the heartbeat cannot + * refresh the lock anymore, it will interrupt itself. + */ + private static class MaterializationRebuildLockHeartbeater implements Runnable { + + private final DbTxnManager txnMgr; + private final String dbName; + private final String tableName; + private final String queryId; + private final long txnId; + private final AtomicReference> task; + + MaterializationRebuildLockHeartbeater(DbTxnManager txnMgr, String dbName, String tableName, + String queryId, long txnId) { + this.txnMgr = txnMgr; + this.queryId = queryId; + this.dbName = dbName; + this.tableName = tableName; + this.txnId = txnId; + this.task = new AtomicReference<>(); + } + + /** + * Send a heartbeat to the metastore for locks and transactions. + */ + @Override + public void run() { + LOG.trace("Heartbeating materialization rebuild lock for {} for query: {}", + AcidUtils.getFullTableName(dbName, tableName), queryId); + boolean refreshed; + try { + refreshed = txnMgr.heartbeatMaterializationRebuildLock(dbName, tableName, txnId); + } catch (LockException e) { + LOG.error("Failed trying to acquire lock", e); + throw new RuntimeException(e); + } + if (!refreshed) { + // We could not heartbeat the lock, i.e., the operation has finished, + // hence we interrupt this work + ScheduledFuture t = task.get(); + if (t != null) { + t.cancel(false); + LOG.debug("Stopped heartbeat for materialization rebuild lock for {} for query: {}", + AcidUtils.getFullTableName(dbName, tableName), queryId); + } + } + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java index 490f3b8ffe..e0e235b46a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java @@ -19,6 +19,7 @@ import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.ValidTxnWriteIdList; +import org.apache.hadoop.hive.metastore.api.LockResponse; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.Driver.LockedDriverState; import org.apache.hadoop.hive.ql.QueryPlan; @@ -269,4 +270,14 @@ * Even a single statement, (e.g. Merge, multi-insert may generates several writes). */ int getStmtIdAndIncrement(); + + /** + * Acquire the materialization rebuild lock for a given view. We need to specify the fully + * qualified name of the materialized view and the open transaction ID so we can identify + * uniquely the lock. + * @return the response from the metastore, where the lock id is equal to the txn id and + * the status can be either ACQUIRED or NOT ACQUIRED + */ + LockResponse acquireMaterializationRebuildLock(String dbName, String tableName, long txnId) + throws LockException; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java index c8cafa2a68..623b0376c6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java @@ -24,6 +24,8 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.LockResponse; +import org.apache.hadoop.hive.metastore.api.LockState; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.Driver.LockedDriverState; import org.apache.hadoop.hive.ql.QueryPlan; @@ -213,4 +215,11 @@ public boolean isImplicitTransactionOpen() { return true; } + @Override + public LockResponse acquireMaterializationRebuildLock(String dbName, String tableName, long txnId) + throws LockException { + // This is default implementation. Locking only works for incremental maintenance + // which only works for DB transactional manager, thus we cannot acquire a lock. + return new LockResponse(0L, LockState.NOT_ACQUIRED); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 38fbb7ba53..2dd1d35a12 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -59,9 +59,22 @@ import javax.jdo.JDODataStoreException; +import com.google.common.collect.ImmutableList; import org.apache.calcite.plan.RelOptMaterialization; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.RelBuilder; import org.apache.commons.io.FilenameUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileChecksum; @@ -76,6 +89,8 @@ import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.common.ValidTxnWriteIdList; +import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.hive.common.classification.InterfaceStability.Unstable; import org.apache.hadoop.hive.common.log.InPlaceUpdate; @@ -161,7 +176,9 @@ import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveAugmentMaterializationRule; import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils; import org.apache.hadoop.hive.ql.plan.AddPartitionDesc; import org.apache.hadoop.hive.ql.plan.DropTableDesc; @@ -177,6 +194,7 @@ import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; +import org.apache.hive.common.util.TxnIdUtils; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1316,14 +1334,11 @@ public Table apply(org.apache.hadoop.hive.metastore.api.Table table) { * @return the list of materialized views available for rewriting * @throws HiveException */ - public List getValidMaterializedViews(boolean materializedViewRebuild) throws HiveException { - final long defaultDiff = - HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW, - TimeUnit.MILLISECONDS); - final long currentTime = System.currentTimeMillis(); + public List getAllValidMaterializedViews(boolean forceMVContentsUpToDate, ValidTxnWriteIdList txnList) + throws HiveException { + // Final result + List result = new ArrayList<>(); try { - // Final result - List result = new ArrayList<>(); for (String dbName : getMSC().getAllDatabases()) { // From metastore (for security) List materializedViewNames = getMaterializedViewsForRewriting(dbName); @@ -1331,83 +1346,137 @@ public Table apply(org.apache.hadoop.hive.metastore.api.Table table) { // Bail out: empty list continue; } - List materializedViewTables = getTableObjects(dbName, materializedViewNames); - Map databaseInvalidationInfo = - getMSC().getMaterializationsInvalidationInfo(dbName, materializedViewNames); - for (Table materializedViewTable : materializedViewTables) { + result.addAll(getValidMaterializedViews(dbName, materializedViewNames, forceMVContentsUpToDate, txnList)); + } + return result; + } catch (Exception e) { + throw new HiveException(e); + } + } + + public List getValidMaterializedView(String dbName, String materializedViewName, + boolean forceMVContentsUpToDate, ValidTxnWriteIdList txnList) throws HiveException { + return getValidMaterializedViews(dbName, ImmutableList.of(materializedViewName), forceMVContentsUpToDate, txnList); + } + + private List getValidMaterializedViews(String dbName, List materializedViewNames, + boolean forceMVContentsUpToDate, ValidTxnWriteIdList txnList) throws HiveException { + final boolean tryIncrementalRewriting = + HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_INCREMENTAL); + final long defaultDiff = + HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW, + TimeUnit.MILLISECONDS); + final long currentTime = System.currentTimeMillis(); + try { + // Final result + List result = new ArrayList<>(); + List
materializedViewTables = getTableObjects(dbName, materializedViewNames); + Map databaseInvalidationInfo = + getMSC().getMaterializationsInvalidationInfo(dbName, materializedViewNames); + for (Table materializedViewTable : materializedViewTables) { + // Check if materialization defined its own invalidation time window + String timeWindowString = materializedViewTable.getProperty(MATERIALIZED_VIEW_REWRITING_TIME_WINDOW); + long diff = org.apache.commons.lang.StringUtils.isEmpty(timeWindowString) ? defaultDiff : + HiveConf.toTime(timeWindowString, + HiveConf.getDefaultTimeUnit(HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW), + TimeUnit.MILLISECONDS); + Materialization materializationInvInfo = null; + boolean outdated = false; + if (diff < 0L) { + // We only consider the materialized view to be outdated if forceOutdated = true, i.e., + // if it is a rebuild. Otherwise, it passed the test and we use it as it is. + outdated = forceMVContentsUpToDate; + } else { // Check whether the materialized view is invalidated - Materialization materializationInvalidationInfo = + materializationInvInfo = databaseInvalidationInfo.get(materializedViewTable.getTableName()); - if (materializationInvalidationInfo == null) { + if (materializationInvInfo == null) { LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + " ignored for rewriting as there was no information loaded in the invalidation cache"); continue; } - // Check if materialization defined its own invalidation time window - String timeWindowString = materializedViewTable.getProperty(MATERIALIZED_VIEW_REWRITING_TIME_WINDOW); - long diff = org.apache.commons.lang.StringUtils.isEmpty(timeWindowString) ? defaultDiff : - HiveConf.toTime(timeWindowString, - HiveConf.getDefaultTimeUnit(HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW), - TimeUnit.MILLISECONDS); - - long invalidationTime = materializationInvalidationInfo.getInvalidationTime(); + long invalidationTime = materializationInvInfo.getInvalidationTime(); + if (invalidationTime == Long.MIN_VALUE) { + LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + + " ignored for rewriting as it contains non-transactional tables"); + continue; + } // If the limit is not met, we do not add the materialized view. // If we are doing a rebuild, we do not consider outdated materialized views either. - if (diff == 0L || materializedViewRebuild) { + if (diff == 0L || forceMVContentsUpToDate) { if (invalidationTime != 0L) { - // If parameter is zero, materialized view cannot be outdated at all - LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + - " ignored for rewriting as its contents are outdated"); - continue; + outdated = true; } } else { - if (invalidationTime != 0 && invalidationTime > currentTime - diff) { - LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + - " ignored for rewriting as its contents are outdated"); - continue; + if (invalidationTime != 0L && invalidationTime > currentTime - diff) { + outdated = true; } } + } - // It passed the test, load - RelOptMaterialization materialization = - HiveMaterializedViewsRegistry.get().getRewritingMaterializedView( - dbName, materializedViewTable.getTableName()); - if (materialization != null) { - RelNode viewScan = materialization.tableRel; - RelOptHiveTable cachedMaterializedViewTable; - if (viewScan instanceof Project) { - // There is a Project on top (due to nullability) - cachedMaterializedViewTable = (RelOptHiveTable) viewScan.getInput(0).getTable(); - } else { - cachedMaterializedViewTable = (RelOptHiveTable) viewScan.getTable(); - } - if (cachedMaterializedViewTable.getHiveTableMD().getCreateTime() == - materializedViewTable.getCreateTime()) { - // It is in the cache and up to date - result.add(materialization); - continue; + if (outdated && (!tryIncrementalRewriting || materializationInvInfo == null + || txnList == null || materializationInvInfo.isSourceTablesUpdateDeleteModified())) { + // We will not try partial rewriting either because the config specification, this + // is a rebuild over some non-transactional table, or there were update/delete + // operations in the source tables (not supported yet) + LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + + " ignored for rewriting as its contents are outdated"); + continue; + } + + // It passed the test, load + RelOptMaterialization materialization = + HiveMaterializedViewsRegistry.get().getRewritingMaterializedView( + dbName, materializedViewTable.getTableName()); + if (materialization != null) { + RelNode viewScan = materialization.tableRel; + RelOptHiveTable cachedMaterializedViewTable; + if (viewScan instanceof Project) { + // There is a Project on top (due to nullability) + cachedMaterializedViewTable = (RelOptHiveTable) viewScan.getInput(0).getTable(); + } else { + cachedMaterializedViewTable = (RelOptHiveTable) viewScan.getTable(); + } + if (cachedMaterializedViewTable.getHiveTableMD().getCreateTime() == + materializedViewTable.getCreateTime()) { + // It is in the cache and up to date + if (outdated) { + // We will rewrite it to include the filters on transaction list + // so we can produce partial rewritings + materialization = augmentMaterializationWithTimeInformation( + materialization, txnList, new ValidTxnWriteIdList( + materializationInvInfo.getValidTxnList())); } + result.add(materialization); + continue; } + } - // It was not present in the cache (maybe because it was added by another HS2) - // or it is not up to date. - if (HiveMaterializedViewsRegistry.get().isInitialized()) { - // But the registry was fully initialized, thus we need to add it - if (LOG.isDebugEnabled()) { - LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + - " was not in the cache"); - } - materialization = HiveMaterializedViewsRegistry.get().createMaterializedView( - conf, materializedViewTable); - if (materialization != null) { - result.add(materialization); - } - } else { - // Otherwise the registry has not been initialized, skip for the time being - if (LOG.isWarnEnabled()) { - LOG.info("Materialized view " + materializedViewTable.getFullyQualifiedName() + " was skipped " - + "because cache has not been loaded yet"); + // It was not present in the cache (maybe because it was added by another HS2) + // or it is not up to date. + if (HiveMaterializedViewsRegistry.get().isInitialized()) { + // But the registry was fully initialized, thus we need to add it + if (LOG.isDebugEnabled()) { + LOG.debug("Materialized view " + materializedViewTable.getFullyQualifiedName() + + " was not in the cache"); + } + materialization = HiveMaterializedViewsRegistry.get().createMaterializedView( + conf, materializedViewTable); + if (materialization != null) { + if (outdated) { + // We will rewrite it to include the filters on transaction list + // so we can produce partial rewritings + materialization = augmentMaterializationWithTimeInformation( + materialization, txnList, new ValidTxnWriteIdList( + materializationInvInfo.getValidTxnList())); } + result.add(materialization); + } + } else { + // Otherwise the registry has not been initialized, skip for the time being + if (LOG.isWarnEnabled()) { + LOG.info("Materialized view " + materializedViewTable.getFullyQualifiedName() + " was skipped " + + "because cache has not been loaded yet"); } } } @@ -1417,6 +1486,24 @@ public Table apply(org.apache.hadoop.hive.metastore.api.Table table) { } } + /** + * Method to enrich the materialization query contained in the input with + * its invalidation. + */ + private static RelOptMaterialization augmentMaterializationWithTimeInformation( + RelOptMaterialization materialization, ValidTxnWriteIdList currentTxnList, + ValidTxnWriteIdList materializationTxnList) { + final RexBuilder rexBuilder = materialization.queryRel.getCluster().getRexBuilder(); + final HepProgramBuilder augmentMaterializationProgram = new HepProgramBuilder() + .addRuleInstance(new HiveAugmentMaterializationRule(rexBuilder, currentTxnList, materializationTxnList)); + final HepPlanner augmentMaterializationPlanner = new HepPlanner( + augmentMaterializationProgram.build()); + augmentMaterializationPlanner.setRoot(materialization.queryRel); + final RelNode modifiedQueryRel = augmentMaterializationPlanner.findBestExp(); + return new RelOptMaterialization(materialization.tableRel, modifiedQueryRel, + null, materialization.qualifiedTableName); + } + /** * Get materialized views for the specified database that have enabled rewriting. * @param dbName @@ -2424,8 +2511,6 @@ public Partition getPartition(Table tbl, Map partSpec, * created * @param partPath the path where the partition data is located * @param inheritTableSpecs whether to copy over the table specs for if/of/serde - * @param newFiles An optional list of new files that were moved into this partition. If - * non-null these will be included in the DML event sent to the metastore. * @return result partition object or null if there is no partition * @throws HiveException */ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index 53dc8ec197..6e585e52a5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -193,6 +193,8 @@ public RelOptMaterialization createMaterializedView(HiveConf conf, Table materia private RelOptMaterialization addMaterializedView(HiveConf conf, Table materializedViewTable, OpType opType) { // Bail out if it is not enabled for rewriting if (!materializedViewTable.isRewriteEnabled()) { + LOG.debug("Materialized view " + materializedViewTable.getCompleteName() + + " ignored; it is not rewrite enabled"); return null; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAggregateIncrementalRewritingRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAggregateIncrementalRewritingRule.java new file mode 100644 index 0000000000..aabd75ee1f --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAggregateIncrementalRewritingRule.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules.views; + +import com.google.common.collect.ImmutableList; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.Union; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; +import org.apache.hadoop.hive.ql.optimizer.calcite.functions.HiveSqlMinMaxAggFunction; +import org.apache.hadoop.hive.ql.optimizer.calcite.functions.HiveSqlSumAggFunction; + +import java.util.ArrayList; +import java.util.List; + +/** + * This rule will perform a rewriting to prepare the plan for incremental + * view maintenance in case there exist aggregation operator, so we can + * avoid the INSERT OVERWRITE and use a MERGE statement instead. + * + * In particular, the INSERT OVERWRITE maintenance will look like this + * (in SQL): + * INSERT OVERWRITE mv + * SELECT a, b, SUM(s) as s, SUM(c) AS c + * FROM ( + * SELECT * from mv --OLD DATA + * UNION ALL + * SELECT a, b, SUM(x) AS s, COUNT(*) AS c --NEW DATA + * FROM TAB_A + * JOIN TAB_B ON (TAB_A.a = TAB_B.z) + * WHERE TAB_A.ROW_ID > 5 + * GROUP BY a, b) inner_subq + * GROUP BY a, b; + * + * We need to transform that into: + * MERGE INTO mv + * USING ( + * SELECT a, b, SUM(x) AS s, COUNT(*) AS c --NEW DATA + * FROM TAB_A + * JOIN TAB_B ON (TAB_A.a = TAB_B.z) + * WHERE TAB_A.ROW_ID > 5 + * GROUP BY a, b) source + * ON (mv.a = source.a AND mv.b = source.b) + * WHEN MATCHED AND mv.c + source.c <> 0 + * THEN UPDATE SET mv.s = mv.s + source.s, mv.c = mv.c + source.c + * WHEN NOT MATCHED + * THEN INSERT VALUES (source.a, source.b, s, c); + * + * To be precise, we need to convert it into a MERGE rewritten as: + * FROM mv right outer join _source_ source + * ON (mv.a = source.a AND mv.b = source.b) + * INSERT INTO TABLE mv + * SELECT source.a, source.b, s, c + * WHERE mv.a IS NULL AND mv2.b IS NULL + * INSERT INTO TABLE mv + * SELECT mv.ROW__ID, source.a, source.b, mv.s + source.s, mv.c + source.c + * WHERE source.a=mv.a AND source.b=mv.b + * SORT BY mv.ROW__ID; + */ +public class HiveAggregateIncrementalRewritingRule extends RelOptRule { + + public static final HiveAggregateIncrementalRewritingRule INSTANCE = + new HiveAggregateIncrementalRewritingRule(); + + private HiveAggregateIncrementalRewritingRule() { + super(operand(Aggregate.class, operand(Union.class, any())), + HiveRelFactories.HIVE_BUILDER, + "HiveAggregateIncrementalRewritingRule"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final Aggregate agg = call.rel(0); + final Union union = call.rel(1); + final RexBuilder rexBuilder = + agg.getCluster().getRexBuilder(); + // 1) First branch is query, second branch is MV + final RelNode joinLeftInput = union.getInput(1); + final RelNode joinRightInput = union.getInput(0); + // 2) Build conditions for join and filter and start adding + // expressions for project operator + List projExprs = new ArrayList<>(); + List joinConjs = new ArrayList<>(); + List filterConjs = new ArrayList<>(); + int groupCount = agg.getGroupCount(); + int totalCount = agg.getGroupCount() + agg.getAggCallList().size(); + for (int leftPos = 0, rightPos = totalCount; + leftPos < groupCount; leftPos++, rightPos++) { + RexNode leftRef = rexBuilder.makeInputRef( + joinLeftInput.getRowType().getFieldList().get(leftPos).getType(), leftPos); + RexNode rightRef = rexBuilder.makeInputRef( + joinRightInput.getRowType().getFieldList().get(leftPos).getType(), rightPos); + projExprs.add(rightRef); + joinConjs.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, + ImmutableList.of(leftRef, rightRef))); + filterConjs.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, + ImmutableList.of(leftRef))); + } + // 3) Add the expressions that correspond to the aggregation + // functions + RexNode caseFilterCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, filterConjs); + for (int i = 0, leftPos = groupCount, rightPos = totalCount + groupCount; + leftPos < totalCount; i++, leftPos++, rightPos++) { + // case when mv2.deptno IS NULL AND mv2.deptname IS NULL then s else source.s + mv2.s end + RexNode leftRef = rexBuilder.makeInputRef( + joinLeftInput.getRowType().getFieldList().get(leftPos).getType(), leftPos); + RexNode rightRef = rexBuilder.makeInputRef( + joinRightInput.getRowType().getFieldList().get(leftPos).getType(), rightPos); + // Generate SQLOperator for merging the aggregations + RexNode elseReturn; + SqlAggFunction aggCall = agg.getAggCallList().get(i).getAggregation(); + switch (aggCall.getKind()) { + case SUM: + // SUM and COUNT are rolled up as SUM, hence SUM represents both here + elseReturn = rexBuilder.makeCall(SqlStdOperatorTable.PLUS, + ImmutableList.of(rightRef, leftRef)); + break; + case MIN: { + RexNode condInnerCase = rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, + ImmutableList.of(rightRef, leftRef)); + elseReturn = rexBuilder.makeCall(SqlStdOperatorTable.CASE, + ImmutableList.of(condInnerCase, rightRef, leftRef)); + } + break; + case MAX: { + RexNode condInnerCase = rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN, + ImmutableList.of(rightRef, leftRef)); + elseReturn = rexBuilder.makeCall(SqlStdOperatorTable.CASE, + ImmutableList.of(condInnerCase, rightRef, leftRef)); + } + break; + default: + throw new AssertionError("Found an aggregation that could not be" + + " recognized: " + aggCall); + } + projExprs.add(rexBuilder.makeCall(SqlStdOperatorTable.CASE, + ImmutableList.of(caseFilterCond, rightRef, elseReturn))); + } + RexNode joinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, joinConjs); + RexNode filterCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, filterConjs); + // 3) Build plan + RelNode newNode = call.builder() + .push(union.getInput(1)) + .push(union.getInput(0)) + .join(JoinRelType.RIGHT, joinCond) + .filter(rexBuilder.makeCall(SqlStdOperatorTable.OR, ImmutableList.of(joinCond, filterCond))) + .project(projExprs) + .build(); + call.transformTo(newNode); + } + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAugmentMaterializationRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAugmentMaterializationRule.java new file mode 100644 index 0000000000..420e6c45ad --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAugmentMaterializationRule.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules.views; + +import com.google.common.collect.ImmutableList; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.RelBuilder; +import org.apache.hadoop.hive.common.ValidTxnWriteIdList; +import org.apache.hadoop.hive.common.ValidWriteIdList; +import org.apache.hadoop.hive.ql.metadata.VirtualColumn; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; +import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; +import org.apache.hive.common.util.TxnIdUtils; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * This rule will rewrite the materialized view with information about + * its invalidation data. In particular, if any of the tables used by the + * materialization has been updated since the materialization was created, + * it will introduce a filter operator on top of that table in the materialization + * definition, making explicit the data contained in it so the rewriting + * algorithm can use this information to rewrite the query as a combination of the + * outdated materialization data and the new original data in the source tables. + * If the data in the source table matches the current data in the snapshot, + * no filter is created. + */ +public class HiveAugmentMaterializationRule extends RelOptRule { + + private final RexBuilder rexBuilder; + private final ValidTxnWriteIdList currentTxnList; + private final ValidTxnWriteIdList materializationTxnList; + private final Set visited; + + public HiveAugmentMaterializationRule(RexBuilder rexBuilder, + ValidTxnWriteIdList currentTxnList, ValidTxnWriteIdList materializationTxnList) { + super(operand(TableScan.class, any()), + HiveRelFactories.HIVE_BUILDER, "HiveAugmentMaterializationRule"); + this.rexBuilder = rexBuilder; + this.currentTxnList = currentTxnList; + this.materializationTxnList = materializationTxnList; + this.visited = new HashSet<>(); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final TableScan tableScan = call.rel(0); + if (!visited.add(tableScan)) { + // Already visited + return; + } + final String tableQName = + ((RelOptHiveTable)tableScan.getTable()).getHiveTableMD().getFullyQualifiedName(); + final ValidWriteIdList tableCurrentTxnList = + currentTxnList.getTableValidWriteIdList(tableQName); + final ValidWriteIdList tableMaterializationTxnList = + materializationTxnList.getTableValidWriteIdList(tableQName); + if (TxnIdUtils.checkEquivalentWriteIds(tableCurrentTxnList, tableMaterializationTxnList)) { + // This table has not been modified since materialization was created, + // nothing to do + return; + } + // ROW__ID: struct + int rowIDPos = tableScan.getTable().getRowType().getField( + VirtualColumn.ROWID.getName(), false, false).getIndex(); + RexNode rowIDFieldAccess = rexBuilder.makeFieldAccess( + rexBuilder.makeInputRef(tableScan.getTable().getRowType().getFieldList().get(rowIDPos).getType(), rowIDPos), + 0); + // Now we create the filter with the transactions information. + // In particular, each table in the materialization will only have contents such that: + // ROW_ID.writeid <= high_watermark and ROW_ID.writeid not in (open/invalid_ids) + // Hence, we add that condition on top of the source table. + // The rewriting will then have the possibility to create partial rewritings that read + // the materialization and the source tables, and hence, produce an incremental + // rebuild that is more efficient than the full rebuild. + final RelBuilder relBuilder = call.builder(); + relBuilder.push(tableScan); + List conds = new ArrayList<>(); + RelDataType bigIntType = relBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT); + final RexNode literalHighWatermark = rexBuilder.makeLiteral( + tableMaterializationTxnList.getHighWatermark(), bigIntType, false); + conds.add( + rexBuilder.makeCall( + SqlStdOperatorTable.LESS_THAN_OR_EQUAL, + ImmutableList.of(rowIDFieldAccess, literalHighWatermark))); + for (long invalidTxn : tableMaterializationTxnList.getInvalidWriteIds()) { + final RexNode literalInvalidTxn = rexBuilder.makeLiteral( + invalidTxn, bigIntType, false); + conds.add( + rexBuilder.makeCall( + SqlStdOperatorTable.NOT_EQUALS, + ImmutableList.of(rowIDFieldAccess, literalInvalidTxn))); + } + relBuilder.filter(conds); + call.transformTo(relBuilder.build()); + } + +} \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveNoAggregateIncrementalRewritingRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveNoAggregateIncrementalRewritingRule.java new file mode 100644 index 0000000000..ea285d07ea --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveNoAggregateIncrementalRewritingRule.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules.views; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Union; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; + +/** + * This rule will perform a rewriting to prepare the plan for incremental + * view maintenance in case there is no aggregation operator, so we can + * avoid the INSERT OVERWRITE and use a INSERT statement instead. + * In particular, it removes the union branch that reads the old data from + * the materialization, and keeps the branch that will read the new data. + */ +public class HiveNoAggregateIncrementalRewritingRule extends RelOptRule { + + public static final HiveNoAggregateIncrementalRewritingRule INSTANCE = + new HiveNoAggregateIncrementalRewritingRule(); + + private HiveNoAggregateIncrementalRewritingRule() { + super(operand(Union.class, any()), + HiveRelFactories.HIVE_BUILDER, "HiveNoAggregateIncrementalRewritingRule"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final Union union = call.rel(0); + // First branch is query, second branch is MV + RelNode newNode = call.builder() + .push(union.getInput(0)) + .convert(union.getRowType(), false) + .build(); + call.transformTo(newNode); + } + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/MaterializedViewRewritingRelVisitor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/MaterializedViewRewritingRelVisitor.java new file mode 100644 index 0000000000..884efd56ce --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/MaterializedViewRewritingRelVisitor.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.rules.views; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.core.Union; +import org.apache.calcite.util.ControlFlowException; +import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class is a helper to check whether a materialized view rebuild + * can be transformed from INSERT OVERWRITE to INSERT INTO. + * + * We are verifying that: + * 1) the rewriting is rooted by legal operators (Filter and Project) + * before reaching a Union operator, + * 2) the left branch uses the MV that we are trying to rebuild and + * legal operators (Filter and Project), and + * 3) the right branch only uses legal operators (i.e., Filter, Project, + * Join, and TableScan) + */ +public class MaterializedViewRewritingRelVisitor extends RelVisitor { + + private static final Logger LOG = LoggerFactory.getLogger(MaterializedViewRewritingRelVisitor.class); + + + private boolean containsAggregate; + private boolean rewritingAllowed; + + public MaterializedViewRewritingRelVisitor() { + this.containsAggregate = false; + this.rewritingAllowed = false; + } + + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof Aggregate) { + this.containsAggregate = true; + // Aggregate mode - it should be followed by union + // that we need to analyze + RelNode input = node.getInput(0); + if (input instanceof Union) { + check((Union) input); + } + } else if (node instanceof Union) { + // Non aggregate mode - analyze union operator + check((Union) node); + } else if (node instanceof Project) { + // Project operator, we can continue + super.visit(node, ordinal, parent); + } + throw new ReturnedValue(false); + } + + private void check(Union union) { + // We found the Union + if (union.getInputs().size() != 2) { + // Bail out + throw new ReturnedValue(false); + } + // First branch should have the query (with write ID filter conditions) + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof TableScan || + node instanceof Filter || + node instanceof Project || + node instanceof Join) { + // We can continue + super.visit(node, ordinal, parent); + } else if (node instanceof Aggregate && containsAggregate) { + // We can continue + super.visit(node, ordinal, parent); + } else { + throw new ReturnedValue(false); + } + } + }.go(union.getInput(0)); + // Second branch should only have the MV + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof TableScan) { + // We can continue + // TODO: Need to check that this is the same MV that we are rebuilding + RelOptHiveTable hiveTable = (RelOptHiveTable) node.getTable(); + if (!hiveTable.getHiveTableMD().isMaterializedView()) { + // If it is not a materialized view, we do not rewrite it + throw new ReturnedValue(false); + } + if (containsAggregate && !AcidUtils.isFullAcidTable(hiveTable.getHiveTableMD())) { + // If it contains an aggregate and it is not a full acid table, + // we do not rewrite it (we need MERGE support) + throw new ReturnedValue(false); + } + } else if (node instanceof Project) { + // We can continue + super.visit(node, ordinal, parent); + } else { + throw new ReturnedValue(false); + } + } + }.go(union.getInput(1)); + // We pass all the checks, we can rewrite + throw new ReturnedValue(true); + } + + /** + * Starts an iteration. + */ + public RelNode go(RelNode p) { + try { + visit(p, 0, null); + } catch (ReturnedValue e) { + // Rewriting cannot be performed + rewritingAllowed = e.value; + } + return p; + } + + public boolean isContainsAggregate() { + return containsAggregate; + } + + public boolean isRewritingAllowed() { + return rewritingAllowed; + } + + + /** + * Exception used to interrupt a visitor walk. + */ + private static class ReturnedValue extends ControlFlowException { + private final boolean value; + + public ReturnedValue(boolean value) { + this.value = value; + } + } + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 41de17fd46..7a7bdea89d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -72,6 +72,7 @@ import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelVisitor; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Filter; @@ -122,11 +123,15 @@ import org.apache.calcite.util.Pair; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.ObjectPair; +import org.apache.hadoop.hive.common.ValidTxnList; +import org.apache.hadoop.hive.common.ValidTxnWriteIdList; import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.conf.HiveConf.StrictChecks; +import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.QueryState; @@ -221,7 +226,10 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveUnionMergeRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveUnionPullUpConstantsRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveWindowingFixRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveAggregateIncrementalRewritingRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveMaterializedViewRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveNoAggregateIncrementalRewritingRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.MaterializedViewRewritingRelVisitor; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTBuilder; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.HiveOpConverter; @@ -407,7 +415,6 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept disableJoinMerge = true; boolean reAnalyzeAST = false; final boolean materializedView = getQB().isMaterializedView(); - final boolean rebuild = materializedView && createVwDesc.isReplace(); try { if (this.conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { @@ -427,9 +434,13 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept ASTNode newAST = getOptimizedAST(); // 1.1. Fix up the query for insert/ctas/materialized views - if (!rebuild) { - // If it is not a MATERIALIZED VIEW...REBUILD - newAST = fixUpAfterCbo(ast, newAST, cboCtx); + newAST = fixUpAfterCbo(ast, newAST, cboCtx); + + // 1.2. Fix up the query for materialization rebuild + if (mvRebuildMode == MaterializationRebuildMode.AGGREGATE_REBUILD) { + fixUpASTAggregateIncrementalRebuild(newAST); + } else if (mvRebuildMode == MaterializationRebuildMode.NO_AGGREGATE_REBUILD) { + fixUpASTNoAggregateIncrementalRebuild(newAST); } // 2. Regen OP plan from optimized AST @@ -440,29 +451,22 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept throw new CalciteViewSemanticException(e.getMessage()); } } else if (cboCtx.type == PreCboCtx.Type.VIEW && materializedView) { - if (rebuild) { - // Use the CREATE MATERIALIZED VIEW...REBUILD - init(false); - setAST(ast); - reAnalyzeViewAfterCbo(ast); - } else { - // Store text of the ORIGINAL QUERY - String originalText = ctx.getTokenRewriteStream().toString( - cboCtx.nodeOfInterest.getTokenStartIndex(), - cboCtx.nodeOfInterest.getTokenStopIndex()); - unparseTranslator.applyTranslations(ctx.getTokenRewriteStream()); - String expandedText = ctx.getTokenRewriteStream().toString( - cboCtx.nodeOfInterest.getTokenStartIndex(), - cboCtx.nodeOfInterest.getTokenStopIndex()); - // Redo create-table/view analysis, because it's not part of - // doPhase1. - // Use the REWRITTEN AST - init(false); - setAST(newAST); - newAST = reAnalyzeViewAfterCbo(newAST); - createVwDesc.setViewOriginalText(originalText); - createVwDesc.setViewExpandedText(expandedText); - } + // Store text of the ORIGINAL QUERY + String originalText = ctx.getTokenRewriteStream().toString( + cboCtx.nodeOfInterest.getTokenStartIndex(), + cboCtx.nodeOfInterest.getTokenStopIndex()); + unparseTranslator.applyTranslations(ctx.getTokenRewriteStream()); + String expandedText = ctx.getTokenRewriteStream().toString( + cboCtx.nodeOfInterest.getTokenStartIndex(), + cboCtx.nodeOfInterest.getTokenStopIndex()); + // Redo create-table/view analysis, because it's not part of + // doPhase1. + // Use the REWRITTEN AST + init(false); + setAST(newAST); + newAST = reAnalyzeViewAfterCbo(newAST); + createVwDesc.setViewOriginalText(originalText); + createVwDesc.setViewExpandedText(expandedText); viewSelect = newAST; viewsExpanded = new ArrayList<>(); viewsExpanded.add(createVwDesc.getViewName()); @@ -963,6 +967,209 @@ Table materializeCTE(String cteName, CTEClause cte) throws HiveException { return table; } + private void fixUpASTAggregateIncrementalRebuild(ASTNode newAST) throws SemanticException { + // Replace INSERT OVERWRITE by MERGE equivalent rewriting. + // Here we need to do this complex AST rewriting that generates the same plan + // that a MERGE clause would generate because CBO does not support MERGE yet. + // TODO: Support MERGE as first class member in CBO to simplify this logic. + // 1) Replace INSERT OVERWRITE by INSERT + ASTNode updateNode = new ASTSearcher().simpleBreadthFirstSearch( + newAST, HiveParser.TOK_QUERY, HiveParser.TOK_INSERT); + ASTNode destinationNode = (ASTNode) updateNode.getChild(0); + ASTNode newInsertInto = (ASTNode) ParseDriver.adaptor.create( + HiveParser.TOK_INSERT_INTO, "TOK_INSERT_INTO"); + newInsertInto.addChildren(destinationNode.getChildren()); + ASTNode destinationParentNode = (ASTNode) destinationNode.getParent(); + int childIndex = destinationNode.childIndex; + destinationParentNode.deleteChild(childIndex); + destinationParentNode.insertChild(childIndex, newInsertInto); + // 1.1) Extract name as we will need it afterwards: + // TOK_DESTINATION TOK_TAB TOK_TABNAME + ASTNode materializationNode = new ASTSearcher().simpleBreadthFirstSearch( + newInsertInto, HiveParser.TOK_INSERT_INTO, HiveParser.TOK_TAB, HiveParser.TOK_TABNAME); + // 2) Copy INSERT branch and duplicate it, the first branch will be the UPDATE + // for the MERGE statement while the new branch will be the INSERT for the + // MERGE statement + ASTNode updateParent = (ASTNode) updateNode.getParent(); + ASTNode insertNode = (ASTNode) ParseDriver.adaptor.dupTree(updateNode); + insertNode.setParent(updateParent); + updateParent.addChild(insertNode); + // 3) Create ROW_ID column in select clause from left input for the RIGHT OUTER JOIN. + // This is needed for the UPDATE clause. Hence, we find the following node: + // TOK_QUERY + // TOK_FROM + // TOK_RIGHTOUTERJOIN + // TOK_SUBQUERY + // TOK_QUERY + // ... + // TOK_INSERT + // ... + // TOK_SELECT + // And then we create the following child node: + // TOK_SELEXPR + // . + // TOK_TABLE_OR_COL + // cmv_mat_view + // ROW__ID + ASTNode subqueryNodeInputROJ = new ASTSearcher().simpleBreadthFirstSearch( + newAST, HiveParser.TOK_QUERY, HiveParser.TOK_FROM, HiveParser.TOK_RIGHTOUTERJOIN, + HiveParser.TOK_SUBQUERY); + ASTNode selectNodeInputROJ = new ASTSearcher().simpleBreadthFirstSearch( + subqueryNodeInputROJ, HiveParser.TOK_SUBQUERY, HiveParser.TOK_QUERY, + HiveParser.TOK_INSERT, HiveParser.TOK_SELECT); + ASTNode selectExprNodeInputROJ = (ASTNode) ParseDriver.adaptor.create( + HiveParser.TOK_SELEXPR, "TOK_SELEXPR"); + ASTNode dotNodeInputROJ = (ASTNode) ParseDriver.adaptor.create( + HiveParser.DOT, "."); + ASTNode columnTokNodeInputROJ = (ASTNode) ParseDriver.adaptor.create( + HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL"); + ASTNode tableNameNodeInputROJ = (ASTNode) ParseDriver.adaptor.create( + HiveParser.Identifier, Warehouse.getQualifiedName( + materializationNode.getChild(0).getText(), + materializationNode.getChild(1).getText())); + ASTNode rowIdNodeInputROJ = (ASTNode) ParseDriver.adaptor.create( + HiveParser.Identifier, VirtualColumn.ROWID.getName()); + ParseDriver.adaptor.addChild(selectNodeInputROJ, selectExprNodeInputROJ); + ParseDriver.adaptor.addChild(selectExprNodeInputROJ, dotNodeInputROJ); + ParseDriver.adaptor.addChild(dotNodeInputROJ, columnTokNodeInputROJ); + ParseDriver.adaptor.addChild(dotNodeInputROJ, rowIdNodeInputROJ); + ParseDriver.adaptor.addChild(columnTokNodeInputROJ, tableNameNodeInputROJ); + // 4) Transform first INSERT branch into an UPDATE + // 4.1) Adding ROW__ID field + ASTNode selectNodeInUpdate = (ASTNode) updateNode.getChild(1); + if (selectNodeInUpdate.getType() != HiveParser.TOK_SELECT) { + throw new SemanticException("TOK_SELECT expected in incremental rewriting"); + } + ASTNode selectExprNodeInUpdate = (ASTNode) ParseDriver.adaptor.dupNode(selectExprNodeInputROJ); + ASTNode dotNodeInUpdate = (ASTNode) ParseDriver.adaptor.dupNode(dotNodeInputROJ); + ASTNode columnTokNodeInUpdate = (ASTNode) ParseDriver.adaptor.dupNode(columnTokNodeInputROJ); + ASTNode tableNameNodeInUpdate = (ASTNode) ParseDriver.adaptor.dupNode(subqueryNodeInputROJ.getChild(1)); + ASTNode rowIdNodeInUpdate = (ASTNode) ParseDriver.adaptor.dupNode(rowIdNodeInputROJ); + ParseDriver.adaptor.addChild(selectExprNodeInUpdate, dotNodeInUpdate); + ParseDriver.adaptor.addChild(dotNodeInUpdate, columnTokNodeInUpdate); + ParseDriver.adaptor.addChild(dotNodeInUpdate, rowIdNodeInUpdate); + ParseDriver.adaptor.addChild(columnTokNodeInUpdate, tableNameNodeInUpdate); + selectNodeInUpdate.insertChild(0, ParseDriver.adaptor.dupTree(selectExprNodeInUpdate)); + // 4.2) Modifying filter condition. The incremental rewriting rule generated an OR + // clause where first disjunct contains the condition for the UPDATE branch. + // TOK_WHERE + // or + // and <- DISJUNCT FOR + // = + // . + // TOK_TABLE_OR_COL + // $hdt$_0 + // a + // . + // TOK_TABLE_OR_COL + // $hdt$_1 + // a + // = + // . + // TOK_TABLE_OR_COL + // $hdt$_0 + // c + // . + // TOK_TABLE_OR_COL + // $hdt$_1 + // c + // and <- DISJUNCT FOR + // TOK_FUNCTION + // isnull + // . + // TOK_TABLE_OR_COL + // $hdt$_0 + // a + // TOK_FUNCTION + // isnull + // . + // TOK_TABLE_OR_COL + // $hdt$_0 + // c + ASTNode whereClauseInUpdate = null; + for (int i = 0; i < updateNode.getChildren().size(); i++) { + if (updateNode.getChild(i).getType() == HiveParser.TOK_WHERE) { + whereClauseInUpdate = (ASTNode) updateNode.getChild(i); + break; + } + } + if (whereClauseInUpdate == null) { + throw new SemanticException("TOK_WHERE expected in incremental rewriting"); + } + if (whereClauseInUpdate.getChild(0).getType() != HiveParser.KW_OR) { + throw new SemanticException("OR clause expected below TOK_WHERE in incremental rewriting"); + } + // We bypass the OR clause and select the first disjunct + ASTNode newCondInUpdate = (ASTNode) whereClauseInUpdate.getChild(0).getChild(0); + ParseDriver.adaptor.setChild(whereClauseInUpdate, 0, newCondInUpdate); + // 4.3) Finally, we add SORT clause, this is needed for the UPDATE. + // TOK_SORTBY + // TOK_TABSORTCOLNAMEASC + // TOK_NULLS_FIRST + // . + // TOK_TABLE_OR_COL + // cmv_basetable_2 + // ROW__ID + ASTNode sortExprNode = (ASTNode) ParseDriver.adaptor.create( + HiveParser.TOK_SORTBY, "TOK_SORTBY"); + ASTNode orderExprNode = (ASTNode) ParseDriver.adaptor.create( + HiveParser.TOK_TABSORTCOLNAMEASC, "TOK_TABSORTCOLNAMEASC"); + ASTNode nullsOrderExprNode = (ASTNode) ParseDriver.adaptor.create( + HiveParser.TOK_NULLS_FIRST, "TOK_NULLS_FIRST"); + ASTNode dotNodeInSort = (ASTNode) ParseDriver.adaptor.dupTree(dotNodeInUpdate); + ParseDriver.adaptor.addChild(updateNode, sortExprNode); + ParseDriver.adaptor.addChild(sortExprNode, orderExprNode); + ParseDriver.adaptor.addChild(orderExprNode, nullsOrderExprNode); + ParseDriver.adaptor.addChild(nullsOrderExprNode, dotNodeInSort); + // 5) Modify INSERT branch condition. In particular, we need to modify the + // WHERE clause and pick up the second disjunct from the OR operation. + ASTNode whereClauseInInsert = null; + for (int i = 0; i < insertNode.getChildren().size(); i++) { + if (insertNode.getChild(i).getType() == HiveParser.TOK_WHERE) { + whereClauseInInsert = (ASTNode) insertNode.getChild(i); + break; + } + } + if (whereClauseInInsert == null) { + throw new SemanticException("TOK_WHERE expected in incremental rewriting"); + } + if (whereClauseInInsert.getChild(0).getType() != HiveParser.KW_OR) { + throw new SemanticException("OR clause expected below TOK_WHERE in incremental rewriting"); + } + // We bypass the OR clause and select the second disjunct + ASTNode newCondInInsert = (ASTNode) whereClauseInInsert.getChild(0).getChild(1); + ParseDriver.adaptor.setChild(whereClauseInInsert, 0, newCondInInsert); + // 6) Now we set some tree properties related to multi-insert + // operation with INSERT/UPDATE + ctx.setOperation(Context.Operation.MERGE); + ctx.addDestNamePrefix(1, Context.DestClausePrefix.UPDATE); + ctx.addDestNamePrefix(2, Context.DestClausePrefix.INSERT); + } + + private void fixUpASTNoAggregateIncrementalRebuild(ASTNode newAST) throws SemanticException { + // Replace INSERT OVERWRITE by INSERT INTO + // AST tree will have this shape: + // TOK_QUERY + // TOK_FROM + // ... + // TOK_INSERT + // TOK_DESTINATION <- THIS TOKEN IS REPLACED BY 'TOK_INSERT_INTO' + // TOK_TAB + // TOK_TABNAME + // default.cmv_mat_view + // TOK_SELECT + // ... + ASTNode dest = new ASTSearcher().simpleBreadthFirstSearch(newAST, HiveParser.TOK_QUERY, + HiveParser.TOK_INSERT, HiveParser.TOK_DESTINATION); + ASTNode newChild = (ASTNode) ParseDriver.adaptor.create( + HiveParser.TOK_INSERT_INTO, "TOK_INSERT_INTO"); + newChild.addChildren(dest.getChildren()); + ASTNode destParent = (ASTNode) dest.getParent(); + int childIndex = dest.childIndex; + destParent.deleteChild(childIndex); + destParent.insertChild(childIndex, newChild); + } + @Override String fixCtasColumnName(String colName) { if (runCBO) { @@ -1847,7 +2054,36 @@ private RelNode applyMaterializedViewRewriting(RelOptPlanner planner, RelNode ba // Add views to planner List materializations = new ArrayList<>(); try { - materializations = Hive.get().getValidMaterializedViews(rewrittenRebuild); + // Extract tables used by the query which will in turn be used to generate + // the corresponding txn write ids + List tablesUsed = new ArrayList<>(); + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof TableScan) { + TableScan ts = (TableScan) node; + tablesUsed.add(((RelOptHiveTable) ts.getTable()).getHiveTableMD().getFullyQualifiedName()); + } + super.visit(node, ordinal, parent); + } + }.go(basePlan); + final String validTxnsList = conf.get(ValidTxnList.VALID_TXNS_KEY); + ValidTxnWriteIdList txnWriteIds = null; + if (validTxnsList != null && !validTxnsList.isEmpty()) { + txnWriteIds = SessionState.get().getTxnMgr().getValidWriteIds(tablesUsed, validTxnsList); + } + if (mvRebuildMode != MaterializationRebuildMode.NONE) { + // We only retrieve the materialization corresponding to the rebuild. In turn, + // we pass 'true' for the forceMVContentsUpToDate parameter, as we cannot allow the + // materialization contents to be stale for a rebuild if we want to use it. + materializations = Hive.get().getValidMaterializedView(mvRebuildDbName, mvRebuildName, + true, txnWriteIds); + } else { + // This is not a rebuild, we retrieve all the materializations. In turn, we do not need + // to force the materialization contents to be up-to-date, as this is not a rebuild, and + // we apply the user parameters (HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW) instead. + materializations = Hive.get().getAllValidMaterializedViews(false, txnWriteIds); + } // We need to use the current cluster for the scan operator on views, // otherwise the planner will throw an Exception (different planners) materializations = Lists.transform(materializations, @@ -1920,6 +2156,29 @@ private RelNode copyNodeScan(RelNode scan) { RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(mdProvider)); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: View-based rewriting"); if (calcitePreMVRewritingPlan != basePlan) { + // A rewriting was produced, we will check whether it was part of an incremental rebuild + // to try to replace INSERT OVERWRITE by INSERT + if (mvRebuildMode == MaterializationRebuildMode.INSERT_OVERWRITE_REBUILD && + HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_INCREMENTAL) && + HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REBUILD_INCREMENTAL)) { + // First we need to check if it is valid to convert to MERGE/INSERT INTO. + // If we succeed, we modify the plan and afterwards the AST. + // MV should be an acid table. + MaterializedViewRewritingRelVisitor visitor = new MaterializedViewRewritingRelVisitor(); + visitor.go(basePlan); + if (visitor.isRewritingAllowed()) { + // Trigger rewriting to remove UNION branch with MV + if (visitor.isContainsAggregate()) { + basePlan = hepPlan(basePlan, false, mdProvider, null, + HepMatchOrder.TOP_DOWN, HiveAggregateIncrementalRewritingRule.INSTANCE); + mvRebuildMode = MaterializationRebuildMode.AGGREGATE_REBUILD; + } else { + basePlan = hepPlan(basePlan, false, mdProvider, null, + HepMatchOrder.TOP_DOWN, HiveNoAggregateIncrementalRewritingRule.INSTANCE); + mvRebuildMode = MaterializationRebuildMode.NO_AGGREGATE_REBUILD; + } + } + } // Now we trigger some needed optimization rules again basePlan = applyPreJoinOrderingTransforms(basePlan, mdProvider, executorProvider); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MaterializedViewRebuildSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MaterializedViewRebuildSemanticAnalyzer.java index 75eb50c579..e5af95b121 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MaterializedViewRebuildSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MaterializedViewRebuildSemanticAnalyzer.java @@ -18,33 +18,19 @@ package org.apache.hadoop.hive.ql.parse; -import org.apache.hadoop.hive.common.HiveStatsUtils; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.conf.HiveVariableSource; -import org.apache.hadoop.hive.conf.VariableSubstitution; -import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.LockState; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryState; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; +import org.apache.hadoop.hive.ql.lockmgr.LockException; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - /** * MaterializedViewRebuildSemanticAnalyzer. * Rewrites ALTER MATERIALIZED VIEW _mv_name_ REBUILD statement into @@ -54,7 +40,6 @@ private static final Logger LOG = LoggerFactory.getLogger(MaterializedViewRebuildSemanticAnalyzer.class); - static final private LogHelper console = new LogHelper(LOG); public MaterializedViewRebuildSemanticAnalyzer(QueryState queryState) throws SemanticException { @@ -64,7 +49,7 @@ public MaterializedViewRebuildSemanticAnalyzer(QueryState queryState) throws Sem @Override public void analyzeInternal(ASTNode ast) throws SemanticException { - if (rewrittenRebuild) { + if (mvRebuildMode != MaterializationRebuildMode.NONE) { super.analyzeInternal(ast); return; } @@ -86,14 +71,35 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { throw new SemanticException(ErrorMsg.MATERIALIZED_VIEW_DEF_EMPTY); } Context ctx = new Context(queryState.getConf()); - rewrittenAST = ParseUtils.parse("insert overwrite table `" + - dbDotTable + "` " + viewText, ctx); + rewrittenAST = ParseUtils.parse("insert overwrite table " + + "`" + qualifiedTableName[0] + "`.`" + qualifiedTableName[1] + "` " + + viewText, ctx); this.ctx.addRewrittenStatementContext(ctx); + + if (!this.ctx.isExplainPlan() && AcidUtils.isTransactionalTable(tab)) { + // Acquire lock for the given materialized view. Only one rebuild per materialized + // view can be triggered at a given time, as otherwise we might produce incorrect + // results if incremental maintenance is triggered. + HiveTxnManager txnManager = SessionState.get().getTxnMgr(); + LockState state; + try { + state = txnManager.acquireMaterializationRebuildLock( + qualifiedTableName[0], qualifiedTableName[1], txnManager.getCurrentTxnId()).getState(); + } catch (LockException e) { + throw new SemanticException("Exception acquiring lock for rebuilding the materialized view", e); + } + if (state != LockState.ACQUIRED) { + throw new SemanticException("Another process is rebuilding the materialized view " + dbDotTable); + } + } } catch (Exception e) { throw new SemanticException(e); } - rewrittenRebuild = true; - LOG.info("Rebuilding view " + dbDotTable); + mvRebuildMode = MaterializationRebuildMode.INSERT_OVERWRITE_REBUILD; + mvRebuildDbName = qualifiedTableName[0]; + mvRebuildName = qualifiedTableName[1]; + + LOG.debug("Rebuilding materialized view " + dbDotTable); super.analyzeInternal(rewrittenAST); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 3b74abacf3..7f0010855b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -346,7 +346,9 @@ protected boolean noscan; // whether this is a mv rebuild rewritten expression - protected boolean rewrittenRebuild = false; + protected MaterializationRebuildMode mvRebuildMode = MaterializationRebuildMode.NONE; + protected String mvRebuildDbName; // Db name for materialization to rebuild + protected String mvRebuildName; // Name for materialization to rebuild protected volatile boolean disableJoinMerge = false; protected final boolean defaultJoinMerge; @@ -2210,7 +2212,7 @@ private void getMetaData(QB qb, ReadEntity parentInput) case HiveParser.TOK_TAB: { TableSpec ts = new TableSpec(db, conf, ast); if (ts.tableHandle.isView() || - (!rewrittenRebuild && ts.tableHandle.isMaterializedView())) { + (mvRebuildMode == MaterializationRebuildMode.NONE && ts.tableHandle.isMaterializedView())) { throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg()); } @@ -14695,4 +14697,11 @@ public void setInvalidQueryMaterializationReason( public boolean isValidQueryMaterialization() { return (invalidQueryMaterializationReason == null); } + + protected enum MaterializationRebuildMode { + NONE, + INSERT_OVERWRITE_REBUILD, + AGGREGATE_REBUILD, + NO_AGGREGATE_REBUILD + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java index 4c86fb8937..fe570f0f8e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java @@ -249,7 +249,7 @@ public PlanFragment createPlanFragment(String query, int num, ApplicationId spli DriverCleanup driverCleanup = new DriverCleanup(driver, txnManager, splitsAppId.toString()); boolean needsCleanup = true; try { - CommandProcessorResponse cpr = driver.compileAndRespond(query); + CommandProcessorResponse cpr = driver.compileAndRespond(query, true); if (cpr.getResponseCode() != 0) { throw new HiveException("Failed to compile query: " + cpr.getException()); } @@ -280,7 +280,7 @@ public PlanFragment createPlanFragment(String query, int num, ApplicationId spli HiveConf.setVar(conf, ConfVars.HIVE_EXECUTION_MODE, "llap"); query = "select * from " + tableName; - cpr = driver.compileAndRespond(query); + cpr = driver.compileAndRespond(query, true); if(cpr.getResponseCode() != 0) { throw new HiveException("Failed to create temp table: "+cpr.getException()); } diff --git a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_4.q b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_4.q index c7f050be8d..dbea060d7a 100644 --- a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_4.q +++ b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_4.q @@ -24,28 +24,30 @@ analyze table cmv_basetable_2 compute statistics for columns; -- CREATE VIEW WITH REWRITE DISABLED EXPLAIN -CREATE MATERIALIZED VIEW cmv_mat_view AS - SELECT cmv_basetable.a, cmv_basetable_2.c +CREATE MATERIALIZED VIEW cmv_mat_view TBLPROPERTIES ('transactional'='true') AS + SELECT cmv_basetable.a, cmv_basetable_2.c, sum(cmv_basetable_2.d) FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) WHERE cmv_basetable_2.c > 10.0 GROUP BY cmv_basetable.a, cmv_basetable_2.c; -CREATE MATERIALIZED VIEW cmv_mat_view AS - SELECT cmv_basetable.a, cmv_basetable_2.c +CREATE MATERIALIZED VIEW cmv_mat_view TBLPROPERTIES ('transactional'='true') AS + SELECT cmv_basetable.a, cmv_basetable_2.c, sum(cmv_basetable_2.d) FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) WHERE cmv_basetable_2.c > 10.0 GROUP BY cmv_basetable.a, cmv_basetable_2.c; +analyze table cmv_mat_view compute statistics for columns; + DESCRIBE FORMATTED cmv_mat_view; -- CANNOT USE THE VIEW, IT IS DISABLED FOR REWRITE EXPLAIN -SELECT cmv_basetable.a +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) WHERE cmv_basetable_2.c > 10.10 GROUP BY cmv_basetable.a, cmv_basetable_2.c; -SELECT cmv_basetable.a +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) WHERE cmv_basetable_2.c > 10.10 GROUP BY cmv_basetable.a, cmv_basetable_2.c; @@ -65,12 +67,12 @@ DESCRIBE FORMATTED cmv_mat_view; -- CANNOT USE THE VIEW, IT IS OUTDATED EXPLAIN -SELECT cmv_basetable.a +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) WHERE cmv_basetable_2.c > 10.10 GROUP BY cmv_basetable.a, cmv_basetable_2.c; -SELECT cmv_basetable.a +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) WHERE cmv_basetable_2.c > 10.10 GROUP BY cmv_basetable.a, cmv_basetable_2.c; @@ -85,12 +87,76 @@ DESCRIBE FORMATTED cmv_mat_view; -- NOW IT CAN BE USED AGAIN EXPLAIN -SELECT cmv_basetable.a +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +-- NOW AN UPDATE +UPDATE cmv_basetable_2 SET a=2 WHERE a=1; + +-- INCREMENTAL REBUILD CANNOT BE TRIGGERED +EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +-- MV CAN BE USED +EXPLAIN +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +-- NOW A DELETE +DELETE FROM cmv_basetable_2 WHERE a=2; + +-- INCREMENTAL REBUILD CANNOT BE TRIGGERED +EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +-- MV CAN BE USED +EXPLAIN +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +-- NOW AN INSERT +insert into cmv_basetable_2 values + (1, 'charlie', 15.8, 1); + +-- INCREMENTAL REBUILD CAN BE TRIGGERED AGAIN +EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +-- MV CAN BE USED +EXPLAIN +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) WHERE cmv_basetable_2.c > 10.10 GROUP BY cmv_basetable.a, cmv_basetable_2.c; -SELECT cmv_basetable.a +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) WHERE cmv_basetable_2.c > 10.10 GROUP BY cmv_basetable.a, cmv_basetable_2.c; diff --git a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_5.q b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_5.q new file mode 100644 index 0000000000..112b44cfd0 --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_5.q @@ -0,0 +1,123 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.materializedview.rewriting=true; + +create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true'); + +insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1); + +analyze table cmv_basetable compute statistics for columns; + +create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true'); + +insert into cmv_basetable_2 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3); + +analyze table cmv_basetable_2 compute statistics for columns; + +CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE + TBLPROPERTIES ('transactional'='true') AS + SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0; +analyze table cmv_mat_view compute statistics for columns; + +insert into cmv_basetable_2 values + (3, 'charlie', 15.8, 1); + +analyze table cmv_basetable_2 compute statistics for columns; + +-- CANNOT USE THE VIEW, IT IS OUTDATED +EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10; + +SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10; + +-- REBUILD +EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +DESCRIBE FORMATTED cmv_mat_view; + +-- NOW IT CAN BE USED AGAIN +EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10; + +SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10; + +-- NOW AN UPDATE +UPDATE cmv_basetable_2 SET a=2 WHERE a=1; + +-- INCREMENTAL REBUILD CANNOT BE TRIGGERED +EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +-- MV CAN BE USED +EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10; + +SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10; + +-- NOW A DELETE +DELETE FROM cmv_basetable_2 WHERE a=2; + +-- INCREMENTAL REBUILD CANNOT BE TRIGGERED +EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +-- MV CAN BE USED +EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10; + +SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10; + +-- NOW AN INSERT +insert into cmv_basetable_2 values + (1, 'charlie', 15.8, 1); + +-- INCREMENTAL REBUILD CAN BE TRIGGERED AGAIN +EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +-- MV CAN BE USED +EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10; + +SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10; + +drop materialized view cmv_mat_view; diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out index 1ef7b876d8..bd0f903008 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out @@ -452,9 +452,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Map 8 <- Union 4 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -476,26 +478,47 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: may be used (ACID table) - Map 5 + Map 7 Map Operator Tree: TableScan alias: cmv_basetable_2 Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((c > 10) and a is not null) (type: boolean) + predicate: ((ROW__ID.writeid > 1) and (c > 10) and a is not null) (type: boolean) Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: decimal(10,2)) Execution mode: llap LLAP IO: may be used (ACID table) + Map 8 + Map Operator Tree: + TableScan + alias: default.cmv_mat_view + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col1 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL + Execution mode: llap + LLAP IO: all inputs Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -525,9 +548,27 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col1 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -536,31 +577,33 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: decimal(10,2)) outputColumnNames: a, c - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll') mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: struct), _col1 (type: struct) - Reducer 4 + Reducer 6 Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 4 + Vertex: Union 4 Stage: Stage-2 Dependency Collection @@ -590,14 +633,16 @@ PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD PREHOOK: type: QUERY PREHOOK: Input: default@cmv_basetable PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view PREHOOK: Output: default@cmv_mat_view POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD POSTHOOK: type: QUERY POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view POSTHOOK: Output: default@cmv_mat_view -POSTHOOK: Lineage: cmv_mat_view.a SIMPLE [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), ] -POSTHOOK: Lineage: cmv_mat_view.c SIMPLE [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.a EXPRESSION [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), (cmv_mat_view)default.cmv_mat_view.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.c EXPRESSION [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), (cmv_mat_view)default.cmv_mat_view.FieldSchema(name:c, type:decimal(10,2), comment:null), ] PREHOOK: query: EXPLAIN SELECT cmv_basetable.a FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out new file mode 100644 index 0000000000..2f880371b3 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out @@ -0,0 +1,1784 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.d SCRIPT [] +PREHOOK: query: analyze table cmv_basetable compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: default@cmv_basetable +#### A masked pattern was here #### +POSTHOOK: query: analyze table cmv_basetable compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: default@cmv_basetable +#### A masked pattern was here #### +PREHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable_2 +PREHOOK: query: insert into cmv_basetable_2 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: insert into cmv_basetable_2 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable_2 +POSTHOOK: Lineage: cmv_basetable_2.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.c SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.d SCRIPT [] +PREHOOK: query: analyze table cmv_basetable_2 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: default@cmv_basetable_2 +#### A masked pattern was here #### +POSTHOOK: query: analyze table cmv_basetable_2 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: default@cmv_basetable_2 +#### A masked pattern was here #### +PREHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW cmv_mat_view TBLPROPERTIES ('transactional'='true') AS + SELECT cmv_basetable.a, cmv_basetable_2.c, sum(cmv_basetable_2.d) + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW cmv_mat_view TBLPROPERTIES ('transactional'='true') AS + SELECT cmv_basetable.a, cmv_basetable_2.c, sum(cmv_basetable_2.d) + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2, Stage-0 + Stage-3 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 2 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)), d (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)), _col2 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2, _col3 + Statistics: Num rows: 3 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col3) + keys: _col0 (type: int), _col2 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Create View Operator: + Create View + columns: a int, c decimal(10,2), _c2 bigint + table properties: + transactional true + expanded text: SELECT `cmv_basetable`.`a`, `cmv_basetable_2`.`c`, sum(`cmv_basetable_2`.`d`) + FROM `default`.`cmv_basetable` JOIN `default`.`cmv_basetable_2` ON (`cmv_basetable`.`a` = `cmv_basetable_2`.`a`) + WHERE `cmv_basetable_2`.`c` > 10.0 + GROUP BY `cmv_basetable`.`a`, `cmv_basetable_2`.`c` + name: default.cmv_mat_view + original text: SELECT cmv_basetable.a, cmv_basetable_2.c, sum(cmv_basetable_2.d) + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c + + Stage: Stage-3 + Stats Work + Basic Stats Work: + + Stage: Stage-5 + Materialized View Work + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view TBLPROPERTIES ('transactional'='true') AS + SELECT cmv_basetable.a, cmv_basetable_2.c, sum(cmv_basetable_2.d) + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view TBLPROPERTIES ('transactional'='true') AS + SELECT cmv_basetable.a, cmv_basetable_2.c, sum(cmv_basetable_2.d) + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: analyze table cmv_mat_view compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: analyze table cmv_mat_view compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view +#### A masked pattern was here #### +PREHOOK: query: DESCRIBE FORMATTED cmv_mat_view +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cmv_mat_view +POSTHOOK: query: DESCRIBE FORMATTED cmv_mat_view +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cmv_mat_view +# col_name data_type comment +a int +c decimal(10,2) +_c2 bigint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"_c2\":\"true\",\"a\":\"true\",\"c\":\"true\"}} + numFiles 2 + numRows 2 + rawDataSize 248 + totalSize 706 + transactional true + transactional_properties default +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: SELECT cmv_basetable.a, cmv_basetable_2.c, sum(cmv_basetable_2.d) + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c +View Expanded Text: SELECT `cmv_basetable`.`a`, `cmv_basetable_2`.`c`, sum(`cmv_basetable_2`.`d`) + FROM `default`.`cmv_basetable` JOIN `default`.`cmv_basetable_2` ON (`cmv_basetable`.`a` = `cmv_basetable_2`.`a`) + WHERE `cmv_basetable_2`.`c` > 10.0 + GROUP BY `cmv_basetable`.`a`, `cmv_basetable_2`.`c` +View Rewrite Enabled: No +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 2 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((c > 10.1) and a is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)), d (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)), _col2 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2, _col3 + Statistics: Num rows: 3 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col3) + keys: _col0 (type: int), _col2 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col2 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +#### A masked pattern was here #### +3 6 +1 2 +PREHOOK: query: insert into cmv_basetable_2 values + (3, 'charlie', 15.8, 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: insert into cmv_basetable_2 values + (3, 'charlie', 15.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable_2 +POSTHOOK: Lineage: cmv_basetable_2.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.c SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.d SCRIPT [] +PREHOOK: query: analyze table cmv_basetable_2 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: default@cmv_basetable_2 +#### A masked pattern was here #### +POSTHOOK: query: analyze table cmv_basetable_2 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: default@cmv_basetable_2 +#### A masked pattern was here #### +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE +PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE +POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Alter Materialized View Operator: + Alter Materialized View + name: default.cmv_mat_view + operation: UPDATE_REWRITE_FLAG + +PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE +PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE +POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +POSTHOOK: Input: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: DESCRIBE FORMATTED cmv_mat_view +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cmv_mat_view +POSTHOOK: query: DESCRIBE FORMATTED cmv_mat_view +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cmv_mat_view +# col_name data_type comment +a int +c decimal(10,2) +_c2 bigint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"_c2\":\"true\",\"a\":\"true\",\"c\":\"true\"}} + numFiles 2 + numRows 2 + rawDataSize 248 + totalSize 706 + transactional true + transactional_properties default +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: SELECT cmv_basetable.a, cmv_basetable_2.c, sum(cmv_basetable_2.d) + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c +View Expanded Text: SELECT `cmv_basetable`.`a`, `cmv_basetable_2`.`c`, sum(`cmv_basetable_2`.`d`) + FROM `default`.`cmv_basetable` JOIN `default`.`cmv_basetable_2` ON (`cmv_basetable`.`a` = `cmv_basetable_2`.`a`) + WHERE `cmv_basetable_2`.`c` > 10.0 + GROUP BY `cmv_basetable`.`a`, `cmv_basetable_2`.`c` +View Rewrite Enabled: Yes +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 3 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((c > 10.1) and a is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)), d (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)), _col2 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2, _col3 + Statistics: Num rows: 2 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col3) + keys: _col0 (type: int), _col2 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col2 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +#### A masked pattern was here #### +3 2 +3 6 +1 2 +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-6 depends on stages: Stage-4, Stage-5 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Reducer 4 <- Reducer 2 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.cmv_mat_view + Statistics: Num rows: 2 Data size: 248 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)), _c2 (type: bigint), ROW__ID (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: struct) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 5 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 8 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 3 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((ROW__ID.writeid > 1) and (c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)), d (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)), _col2 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 _col0 (type: int), _col1 (type: decimal(10,2)) + 1 _col0 (type: int), _col1 (type: decimal(10,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 324 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (_col0 is null and _col1 is null) (type: boolean) + Statistics: Num rows: 1 Data size: 324 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col4 (type: int), _col5 (type: decimal(10,2)), CASE WHEN ((_col0 is null and _col1 is null)) THEN (_col6) ELSE ((_col6 + _col2)) END (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(10,2)), _col2 (type: bigint) + outputColumnNames: a, c, _c2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll'), compute_stats(_c2, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Filter Operator + predicate: ((_col0 = _col4) and (_col1 = _col5)) (type: boolean) + Statistics: Num rows: 1 Data size: 324 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: struct), _col4 (type: int), _col5 (type: decimal(10,2)), CASE WHEN ((_col0 is null and _col1 is null)) THEN (_col6) ELSE ((_col6 + _col2)) END (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int), _col2 (type: decimal(10,2)), _col3 (type: bigint) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), VALUE._col1 (type: decimal(10,2)), VALUE._col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: UPDATE + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2, _col3 + Statistics: Num rows: 2 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col3) + keys: _col0 (type: int), _col2 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + + Stage: Stage-4 + Stats Work + Basic Stats Work: + + Stage: Stage-6 + Materialized View Work + + Stage: Stage-1 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: UPDATE + + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, c, _c2 + Column Types: int, decimal(10,2), bigint + Table: default.cmv_mat_view + +PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view +POSTHOOK: Lineage: cmv_mat_view._c2 EXPRESSION [(cmv_mat_view)default.cmv_mat_view.FieldSchema(name:a, type:int, comment:null), (cmv_mat_view)default.cmv_mat_view.FieldSchema(name:c, type:decimal(10,2), comment:null), (cmv_basetable_2)cmv_basetable_2.FieldSchema(name:d, type:int, comment:null), (cmv_mat_view)default.cmv_mat_view.FieldSchema(name:_c2, type:bigint, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.a SIMPLE [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.c SIMPLE [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), ] +PREHOOK: query: DESCRIBE FORMATTED cmv_mat_view +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cmv_mat_view +POSTHOOK: query: DESCRIBE FORMATTED cmv_mat_view +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cmv_mat_view +# col_name data_type comment +a int +c decimal(10,2) +_c2 bigint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + numFiles 3 + totalSize 1453 + transactional true + transactional_properties default +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: SELECT cmv_basetable.a, cmv_basetable_2.c, sum(cmv_basetable_2.d) + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c +View Expanded Text: SELECT `cmv_basetable`.`a`, `cmv_basetable_2`.`c`, sum(`cmv_basetable_2`.`d`) + FROM `default`.`cmv_basetable` JOIN `default`.`cmv_basetable_2` ON (`cmv_basetable`.`a` = `cmv_basetable_2`.`a`) + WHERE `cmv_basetable_2`.`c` > 10.0 + GROUP BY `cmv_basetable`.`a`, `cmv_basetable_2`.`c` +View Rewrite Enabled: Yes +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.cmv_mat_view + Filter Operator + predicate: (c > 10.1) (type: boolean) + Select Operator + expressions: a (type: int), _c2 (type: bigint) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +3 6 +1 2 +3 2 +PREHOOK: query: UPDATE cmv_basetable_2 SET a=2 WHERE a=1 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: UPDATE cmv_basetable_2 SET a=2 WHERE a=1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: default@cmv_basetable_2 +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 60 Data size: 7200 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 20 Data size: 2400 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)), d (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 20 Data size: 2400 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 20 Data size: 2400 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)), _col2 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2, _col3 + Statistics: Num rows: 33 Data size: 3960 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col3) + keys: _col0 (type: int), _col2 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(10,2)), _col2 (type: bigint) + outputColumnNames: a, c, _c2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll'), compute_stats(_c2, 'hll') + mode: complete + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, c, _c2 + Column Types: int, decimal(10,2), bigint + Table: default.cmv_mat_view + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: default@cmv_mat_view +POSTHOOK: Lineage: cmv_mat_view._c2 EXPRESSION [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:d, type:int, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.a SIMPLE [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.c SIMPLE [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), ] +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.cmv_mat_view + Filter Operator + predicate: (c > 10.1) (type: boolean) + Select Operator + expressions: a (type: int), _c2 (type: bigint) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +2 4 +3 2 +3 6 +PREHOOK: query: DELETE FROM cmv_basetable_2 WHERE a=2 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: DELETE FROM cmv_basetable_2 WHERE a=2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: default@cmv_basetable_2 +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 70 Data size: 8400 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 23 Data size: 2760 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)), d (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 23 Data size: 2760 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 23 Data size: 2760 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)), _col2 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2, _col3 + Statistics: Num rows: 38 Data size: 4560 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col3) + keys: _col0 (type: int), _col2 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(10,2)), _col2 (type: bigint) + outputColumnNames: a, c, _c2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll'), compute_stats(_c2, 'hll') + mode: complete + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, c, _c2 + Column Types: int, decimal(10,2), bigint + Table: default.cmv_mat_view + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: default@cmv_mat_view +POSTHOOK: Lineage: cmv_mat_view._c2 EXPRESSION [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:d, type:int, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.a SIMPLE [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.c SIMPLE [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), ] +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.cmv_mat_view + Filter Operator + predicate: (c > 10.1) (type: boolean) + Select Operator + expressions: a (type: int), _c2 (type: bigint) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +3 2 +3 6 +PREHOOK: query: insert into cmv_basetable_2 values + (1, 'charlie', 15.8, 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: insert into cmv_basetable_2 values + (1, 'charlie', 15.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable_2 +POSTHOOK: Lineage: cmv_basetable_2.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.c SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.d SCRIPT [] +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 + Stage-6 depends on stages: Stage-4, Stage-5 + Stage-1 depends on stages: Stage-3 + Stage-5 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-2 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Reducer 4 <- Reducer 2 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.cmv_mat_view + Statistics: Num rows: 2 Data size: 248 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)), _c2 (type: bigint), ROW__ID (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint), _col3 (type: struct) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 5 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 8 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 85 Data size: 10200 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((ROW__ID.writeid > 4) and (c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 9 Data size: 1080 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)), d (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 9 Data size: 1764 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 9 Data size: 1764 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)), _col2 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join 0 to 1 + keys: + 0 _col0 (type: int), _col1 (type: decimal(10,2)) + 1 _col0 (type: int), _col1 (type: decimal(10,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 1 Data size: 324 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (_col0 is null and _col1 is null) (type: boolean) + Statistics: Num rows: 1 Data size: 324 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col4 (type: int), _col5 (type: decimal(10,2)), CASE WHEN ((_col0 is null and _col1 is null)) THEN (_col6) ELSE ((_col6 + _col2)) END (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(10,2)), _col2 (type: bigint) + outputColumnNames: a, c, _c2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll'), compute_stats(_c2, 'hll') + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct) + Filter Operator + predicate: ((_col0 = _col4) and (_col1 = _col5)) (type: boolean) + Statistics: Num rows: 1 Data size: 324 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: struct), _col4 (type: int), _col5 (type: decimal(10,2)), CASE WHEN ((_col0 is null and _col1 is null)) THEN (_col6) ELSE ((_col6 + _col2)) END (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int), _col2 (type: decimal(10,2)), _col3 (type: bigint) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1528 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: int), VALUE._col1 (type: decimal(10,2)), VALUE._col2 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: UPDATE + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2, _col3 + Statistics: Num rows: 15 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col3) + keys: _col0 (type: int), _col2 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 124 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: bigint) + + Stage: Stage-3 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + + Stage: Stage-4 + Stats Work + Basic Stats Work: + + Stage: Stage-6 + Materialized View Work + + Stage: Stage-1 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: UPDATE + + Stage: Stage-5 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, c, _c2 + Column Types: int, decimal(10,2), bigint + Table: default.cmv_mat_view + +PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view +POSTHOOK: Lineage: cmv_mat_view._c2 EXPRESSION [(cmv_mat_view)default.cmv_mat_view.FieldSchema(name:a, type:int, comment:null), (cmv_mat_view)default.cmv_mat_view.FieldSchema(name:c, type:decimal(10,2), comment:null), (cmv_basetable_2)cmv_basetable_2.FieldSchema(name:d, type:int, comment:null), (cmv_mat_view)default.cmv_mat_view.FieldSchema(name:_c2, type:bigint, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.a SIMPLE [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.c SIMPLE [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), ] +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.cmv_mat_view + Filter Operator + predicate: (c > 10.1) (type: boolean) + Select Operator + expressions: a (type: int), _c2 (type: bigint) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a, sum(cmv_basetable_2.d) +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +3 2 +3 6 +1 1 +PREHOOK: query: drop materialized view cmv_mat_view +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: drop materialized view cmv_mat_view +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out new file mode 100644 index 0000000000..d69f962f11 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out @@ -0,0 +1,1106 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.d SCRIPT [] +PREHOOK: query: analyze table cmv_basetable compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: default@cmv_basetable +#### A masked pattern was here #### +POSTHOOK: query: analyze table cmv_basetable compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: default@cmv_basetable +#### A masked pattern was here #### +PREHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable_2 +PREHOOK: query: insert into cmv_basetable_2 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: insert into cmv_basetable_2 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable_2 +POSTHOOK: Lineage: cmv_basetable_2.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.c SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.d SCRIPT [] +PREHOOK: query: analyze table cmv_basetable_2 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: default@cmv_basetable_2 +#### A masked pattern was here #### +POSTHOOK: query: analyze table cmv_basetable_2 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: default@cmv_basetable_2 +#### A masked pattern was here #### +PREHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE + TBLPROPERTIES ('transactional'='true') AS + SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE + TBLPROPERTIES ('transactional'='true') AS + SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: analyze table cmv_mat_view compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: analyze table cmv_mat_view compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view +#### A masked pattern was here #### +PREHOOK: query: insert into cmv_basetable_2 values + (3, 'charlie', 15.8, 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: insert into cmv_basetable_2 values + (3, 'charlie', 15.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable_2 +POSTHOOK: Lineage: cmv_basetable_2.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.c SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.d SCRIPT [] +PREHOOK: query: analyze table cmv_basetable_2 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: default@cmv_basetable_2 +#### A masked pattern was here #### +POSTHOOK: query: analyze table cmv_basetable_2 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: default@cmv_basetable_2 +#### A masked pattern was here #### +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 3 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((c > 10.1) and a is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +#### A masked pattern was here #### +1 +3 +3 +3 +3 +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((ROW__ID.writeid > 1) and (c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col2 (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + outputColumnNames: a, c + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, c + Column Types: int, decimal(10,2) + Table: default.cmv_mat_view + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: default@cmv_mat_view +POSTHOOK: Lineage: cmv_mat_view.a SIMPLE [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.c SIMPLE [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), ] +PREHOOK: query: DESCRIBE FORMATTED cmv_mat_view +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cmv_mat_view +POSTHOOK: query: DESCRIBE FORMATTED cmv_mat_view +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cmv_mat_view +# col_name data_type comment +a int +c decimal(10,2) + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + numFiles 2 + totalSize 1055 + transactional true + transactional_properties default +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 +View Expanded Text: SELECT `cmv_basetable`.`a`, `cmv_basetable_2`.`c` + FROM `default`.`cmv_basetable` JOIN `default`.`cmv_basetable_2` ON (`cmv_basetable`.`a` = `cmv_basetable_2`.`a`) + WHERE `cmv_basetable_2`.`c` > 10.0 +View Rewrite Enabled: Yes +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.cmv_mat_view + Filter Operator + predicate: (c > 10.1) (type: boolean) + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +1 +3 +3 +3 +3 +PREHOOK: query: UPDATE cmv_basetable_2 SET a=2 WHERE a=1 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: UPDATE cmv_basetable_2 SET a=2 WHERE a=1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: default@cmv_basetable_2 +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 60 Data size: 6960 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 20 Data size: 2320 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 2320 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 20 Data size: 2320 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 33 Data size: 3828 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col2 (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 33 Data size: 3828 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 33 Data size: 3828 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + outputColumnNames: a, c + Statistics: Num rows: 33 Data size: 3828 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, c + Column Types: int, decimal(10,2) + Table: default.cmv_mat_view + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: default@cmv_mat_view +POSTHOOK: Lineage: cmv_mat_view.a SIMPLE [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.c SIMPLE [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), ] +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.cmv_mat_view + Filter Operator + predicate: (c > 10.1) (type: boolean) + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +2 +2 +3 +3 +3 +3 +PREHOOK: query: DELETE FROM cmv_basetable_2 WHERE a=2 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: DELETE FROM cmv_basetable_2 WHERE a=2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: default@cmv_basetable_2 +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 70 Data size: 8120 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 23 Data size: 2668 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 23 Data size: 2668 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 23 Data size: 2668 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 38 Data size: 4408 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col2 (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 38 Data size: 4408 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 38 Data size: 4408 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + outputColumnNames: a, c + Statistics: Num rows: 38 Data size: 4408 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, c + Column Types: int, decimal(10,2) + Table: default.cmv_mat_view + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: default@cmv_mat_view +POSTHOOK: Lineage: cmv_mat_view.a SIMPLE [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.c SIMPLE [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), ] +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.cmv_mat_view + Filter Operator + predicate: (c > 10.1) (type: boolean) + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +3 +3 +3 +3 +PREHOOK: query: insert into cmv_basetable_2 values + (1, 'charlie', 15.8, 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: insert into cmv_basetable_2 values + (1, 'charlie', 15.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable_2 +POSTHOOK: Lineage: cmv_basetable_2.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.c SCRIPT [] +POSTHOOK: Lineage: cmv_basetable_2.d SCRIPT [] +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 85 Data size: 9860 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((ROW__ID.writeid > 4) and (c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 9 Data size: 1044 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col2 (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + outputColumnNames: a, c + Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Write Type: INSERT + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, c + Column Types: int, decimal(10,2) + Table: default.cmv_mat_view + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: default@cmv_mat_view +POSTHOOK: Lineage: cmv_mat_view.a SIMPLE [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.c SIMPLE [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), ] +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.cmv_mat_view + Filter Operator + predicate: (c > 10.1) (type: boolean) + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +3 +3 +3 +3 +1 +PREHOOK: query: drop materialized view cmv_mat_view +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: drop materialized view cmv_mat_view +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out index 1ef7b876d8..bd0f903008 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out @@ -452,9 +452,11 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Map 8 <- Union 4 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -476,26 +478,47 @@ STAGE PLANS: Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: may be used (ACID table) - Map 5 + Map 7 Map Operator Tree: TableScan alias: cmv_basetable_2 Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((c > 10) and a is not null) (type: boolean) + predicate: ((ROW__ID.writeid > 1) and (c > 10) and a is not null) (type: boolean) Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: a (type: int), c (type: decimal(10,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: decimal(10,2)) Execution mode: llap LLAP IO: may be used (ACID table) + Map 8 + Map Operator Tree: + TableScan + alias: default.cmv_mat_view + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col1 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL + Execution mode: llap + LLAP IO: all inputs Reducer 2 Execution mode: llap Reduce Operator Tree: @@ -525,9 +548,27 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col1 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -536,31 +577,33 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: decimal(10,2)) outputColumnNames: a, c - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll') mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: struct), _col1 (type: struct) - Reducer 4 + Reducer 6 Execution mode: llap Reduce Operator Tree: Group By Operator aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 4 + Vertex: Union 4 Stage: Stage-2 Dependency Collection @@ -590,14 +633,16 @@ PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD PREHOOK: type: QUERY PREHOOK: Input: default@cmv_basetable PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view PREHOOK: Output: default@cmv_mat_view POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD POSTHOOK: type: QUERY POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view POSTHOOK: Output: default@cmv_mat_view -POSTHOOK: Lineage: cmv_mat_view.a SIMPLE [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), ] -POSTHOOK: Lineage: cmv_mat_view.c SIMPLE [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.a EXPRESSION [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), (cmv_mat_view)default.cmv_mat_view.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.c EXPRESSION [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), (cmv_mat_view)default.cmv_mat_view.FieldSchema(name:c, type:decimal(10,2), comment:null), ] PREHOOK: query: EXPLAIN SELECT cmv_basetable.a FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) diff --git a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_time_window.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out similarity index 53% rename from ql/src/test/results/clientpositive/materialized_view_create_rewrite_time_window.q.out rename to ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out index bfbac3f349..e820e5cde5 100644 --- a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_time_window.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out @@ -89,98 +89,101 @@ POSTHOOK: type: CREATE_MATERIALIZED_VIEW STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-5 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-5 - Stage-6 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-2, Stage-0 + Stage-3 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: a is not null (type: boolean) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - TableScan - alias: cmv_basetable_2 - Statistics: Num rows: 2 Data size: 484 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((c > 10) and a is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), c (type: decimal(10,2)) + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col2 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(10,2)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col2 (type: decimal(10,2)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.cmv_mat_view - - Stage: Stage-0 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Dependency Collection - Stage: Stage-5 + Stage: Stage-4 Create View Operator: Create View columns: a int, c decimal(10,2) @@ -200,9 +203,15 @@ STAGE PLANS: Stats Work Basic Stats Work: - Stage: Stage-6 + Stage: Stage-5 Materialized View Work + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + PREHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view TBLPROPERTIES('rewriting.time.window'='300s') AS SELECT cmv_basetable.a, cmv_basetable_2.c FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) @@ -241,11 +250,11 @@ Retention: 0 Table Type: MATERIALIZED_VIEW Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - numFiles 1 + numFiles 2 numRows 2 rawDataSize 232 rewriting.time.window 300s - totalSize 325 + totalSize 586 #### A masked pattern was here #### # Storage Information @@ -281,91 +290,96 @@ GROUP BY cmv_basetable.a, cmv_basetable_2.c POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: a is not null (type: boolean) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - TableScan - alias: cmv_basetable_2 - Statistics: Num rows: 2 Data size: 484 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((c > 10.1) and a is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), c (type: decimal(10,2)) + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((c > 10.1) and a is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col2 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(10,2)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col2 (type: decimal(10,2)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -389,8 +403,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_basetable_2 #### A masked pattern was here #### -1 3 +1 PREHOOK: query: insert into cmv_basetable_2 values (3, 'charlie', 15.8, 1) PREHOOK: type: QUERY @@ -457,11 +471,11 @@ Retention: 0 Table Type: MATERIALIZED_VIEW Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - numFiles 1 + numFiles 2 numRows 2 rawDataSize 232 rewriting.time.window 300s - totalSize 325 + totalSize 586 #### A masked pattern was here #### # Storage Information @@ -496,36 +510,21 @@ WHERE cmv_basetable_2.c > 10.10 GROUP BY cmv_basetable.a, cmv_basetable_2.c POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.cmv_mat_view - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (c > 10.1) (type: boolean) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: default.cmv_mat_view + Filter Operator + predicate: (c > 10.1) (type: boolean) + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + ListSink PREHOOK: query: SELECT cmv_basetable.a FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) @@ -545,8 +544,8 @@ POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_basetable_2 POSTHOOK: Input: default@cmv_mat_view #### A masked pattern was here #### -1 3 +1 PREHOOK: query: EXPLAIN ALTER MATERIALIZED VIEW cmv_mat_view REBUILD PREHOOK: type: QUERY @@ -557,104 +556,169 @@ STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0, Stage-4 - Stage-6 depends on stages: Stage-3 - Stage-4 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: a is not null (type: boolean) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - TableScan - alias: cmv_basetable_2 - Statistics: Num rows: 3 Data size: 727 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((c > 10) and a is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), c (type: decimal(10,2)) + Tez +#### A masked pattern was here #### + Edges: + Map 8 <- Union 4 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE) + Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 7 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((ROW__ID.writeid > 1) and (c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 8 + Map Operator Tree: + TableScan + alias: default.cmv_mat_view + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col1 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col2 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(10,2)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col2 (type: decimal(10,2)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col1 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + Select Operator + expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + outputColumnNames: a, c + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll') + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 4 + Vertex: Union 4 Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.cmv_mat_view - Select Operator - expressions: _col0 (type: int), _col1 (type: decimal(10,2)) - outputColumnNames: a, c - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll') - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Dependency Collection Stage: Stage-0 Move Operator @@ -674,43 +738,23 @@ STAGE PLANS: Column Types: int, decimal(10,2) Table: default.cmv_mat_view - Stage: Stage-6 - Materialized View Work - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Materialized View Work PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD PREHOOK: type: QUERY PREHOOK: Input: default@cmv_basetable PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view PREHOOK: Output: default@cmv_mat_view POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD POSTHOOK: type: QUERY POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view POSTHOOK: Output: default@cmv_mat_view -POSTHOOK: Lineage: cmv_mat_view.a SIMPLE [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), ] -POSTHOOK: Lineage: cmv_mat_view.c SIMPLE [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.a EXPRESSION [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), (cmv_mat_view)default.cmv_mat_view.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: cmv_mat_view.c EXPRESSION [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), (cmv_mat_view)default.cmv_mat_view.FieldSchema(name:c, type:decimal(10,2), comment:null), ] PREHOOK: query: DESCRIBE FORMATTED cmv_mat_view PREHOOK: type: DESCTABLE PREHOOK: Input: default@cmv_mat_view @@ -729,11 +773,11 @@ Retention: 0 Table Type: MATERIALIZED_VIEW Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"c\":\"true\"}} - numFiles 1 + numFiles 3 numRows 3 rawDataSize 348 rewriting.time.window 300s - totalSize 332 + totalSize 877 #### A masked pattern was here #### # Storage Information @@ -768,36 +812,21 @@ WHERE cmv_basetable_2.c > 10.10 GROUP BY cmv_basetable.a, cmv_basetable_2.c POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.cmv_mat_view - Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (c > 10.1) (type: boolean) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: default.cmv_mat_view + Filter Operator + predicate: (c > 10.1) (type: boolean) + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + ListSink PREHOOK: query: SELECT cmv_basetable.a FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) @@ -817,9 +846,9 @@ POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_basetable_2 POSTHOOK: Input: default@cmv_mat_view #### A masked pattern was here #### -1 3 3 +1 PREHOOK: query: drop materialized view cmv_mat_view PREHOOK: type: DROP_MATERIALIZED_VIEW PREHOOK: Input: default@cmv_mat_view diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_1.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_1.q.out similarity index 68% rename from ql/src/test/results/clientpositive/materialized_view_rewrite_1.q.out rename to ql/src/test/results/clientpositive/llap/materialized_view_rewrite_1.q.out index 75e828fd42..675fe28921 100644 --- a/ql/src/test/results/clientpositive/materialized_view_rewrite_1.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_1.q.out @@ -210,57 +210,71 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 3 Data size: 315 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((empid < 120) and deptno is not null) (type: boolean) - Statistics: Num rows: 3 Data size: 315 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int), deptno (type: int), name (type: varchar(256)), salary (type: float), commission (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 3 Data size: 315 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 Statistics: Num rows: 3 Data size: 315 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: varchar(256)), _col3 (type: float), _col4 (type: int) - TableScan - alias: depts - Statistics: Num rows: 3 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int), name (type: varchar(256)), locationid (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 3 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: varchar(256)), _col2 (type: int) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col5 (type: int), _col0 (type: int), _col2 (type: varchar(256)), _col3 (type: float), _col4 (type: int), _col6 (type: varchar(256)), _col7 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: ((empid < 120) and deptno is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 315 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), deptno (type: int), name (type: varchar(256)), salary (type: float), commission (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 3 Data size: 315 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 315 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: varchar(256)), _col3 (type: float), _col4 (type: int) + Execution mode: llap + LLAP IO: all inputs + Map 3 + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), name (type: varchar(256)), locationid (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: varchar(256)), _col2 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col5 (type: int), _col0 (type: int), _col2 (type: varchar(256)), _col3 (type: float), _col4 (type: int), _col6 (type: varchar(256)), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 594 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -335,56 +349,70 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 5 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: deptno is not null (type: boolean) - Statistics: Num rows: 5 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int), name (type: varchar(256)), salary (type: float), commission (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 5 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 Statistics: Num rows: 5 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: varchar(256)), _col2 (type: float), _col3 (type: int) - TableScan - alias: depts - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: varchar(256)), _col2 (type: float), _col3 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: deptno is not null (type: boolean) + Statistics: Num rows: 5 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), name (type: varchar(256)), salary (type: float), commission (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 5 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: varchar(256)), _col2 (type: float), _col3 (type: int) + Execution mode: llap + LLAP IO: all inputs + Map 3 + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1, _col2, _col3 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: varchar(256)), _col2 (type: float), _col3 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 490 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -456,36 +484,21 @@ select empid deptno from emps join depts using (deptno) where empid = 1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (deptno = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: default.mv1 + Filter Operator + predicate: (deptno = 1) (type: boolean) + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + ListSink PREHOOK: query: select empid deptno from emps join depts using (deptno) where empid = 1 @@ -545,46 +558,57 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: emps - Statistics: Num rows: 5 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (empid > 120) (type: boolean) - Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int), deptno (type: int), name (type: varchar(256)), salary (type: float), commission (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: default.mv1 - Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (empid < 150) (type: boolean) - Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int), deptno (type: int), name (type: varchar(256)), salary (type: float), commission (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: emps + Statistics: Num rows: 5 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (empid > 120) (type: boolean) + Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), deptno (type: int), name (type: varchar(256)), salary (type: float), commission (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 3 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (empid < 150) (type: boolean) + Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), deptno (type: int), name (type: varchar(256)), salary (type: float), commission (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: all inputs + Union 2 + Vertex: Union 2 Stage: Stage-0 Fetch Operator @@ -655,11 +679,9 @@ STAGE PLANS: Processor Tree: TableScan alias: emps - Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: empid (type: int), deptno (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select empid, deptno from emps group by empid, deptno @@ -721,11 +743,9 @@ STAGE PLANS: Processor Tree: TableScan alias: emps - Statistics: Num rows: 5 Data size: 470 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: empid (type: int), name (type: varchar(256)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 470 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select empid, name from emps group by empid, name @@ -787,11 +807,9 @@ STAGE PLANS: Processor Tree: TableScan alias: default.mv1 - Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: name (type: varchar(256)), salary (type: float) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select name, salary from emps group by name, salary @@ -850,38 +868,48 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: name (type: varchar(256)) - outputColumnNames: name - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: name + Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: name (type: varchar(256)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - keys: name (type: varchar(256)) - mode: hash + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col0 (type: varchar(256)) + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: varchar(256)) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -945,38 +973,48 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: name (type: varchar(256)) - outputColumnNames: name - Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: name + Statistics: Num rows: 3 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: name (type: varchar(256)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - keys: name (type: varchar(256)) - mode: hash + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col0 (type: varchar(256)) + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: varchar(256)) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 91 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1041,38 +1079,48 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: name (type: varchar(256)) - outputColumnNames: name - Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: name + Statistics: Num rows: 4 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: name (type: varchar(256)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - keys: name (type: varchar(256)) - mode: hash + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col0 (type: varchar(256)) + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: varchar(256)) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_2.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_2.q.out similarity index 76% rename from ql/src/test/results/clientpositive/materialized_view_rewrite_2.q.out rename to ql/src/test/results/clientpositive/llap/materialized_view_rewrite_2.q.out index 6e8b2e3e65..e8fe69f32c 100644 --- a/ql/src/test/results/clientpositive/materialized_view_rewrite_2.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_2.q.out @@ -211,36 +211,21 @@ join depts using (deptno) where depts.deptno > 20 group by empid, depts.deptno POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (deptno > 20) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: default.mv1 + Filter Operator + predicate: (deptno > 20) (type: boolean) + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + ListSink PREHOOK: query: select empid from emps join depts using (deptno) where depts.deptno > 20 @@ -305,36 +290,21 @@ join depts using (deptno) where depts.deptno > 20 group by empid, depts.deptno POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (deptno > 20) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: default.mv1 + Filter Operator + predicate: (deptno > 20) (type: boolean) + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + ListSink PREHOOK: query: select empid from emps join depts using (deptno) where depts.deptno > 20 @@ -399,36 +369,21 @@ join depts using (deptno) where depts.deptno > 20 group by empid, depts.deptno POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (deptno > 20) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: default.mv1 + Filter Operator + predicate: (deptno > 20) (type: boolean) + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + ListSink PREHOOK: query: select empid from emps join depts using (deptno) where depts.deptno > 20 @@ -493,36 +448,21 @@ join emps using (deptno) where emps.empid > 15 group by depts.deptno, emps.empid POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (empid > 15) (type: boolean) - Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: default.mv1 + Filter Operator + predicate: (empid > 15) (type: boolean) + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + ListSink PREHOOK: query: select depts.deptno from depts join emps using (deptno) where emps.empid > 15 @@ -597,41 +537,51 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (empid > 15) (type: boolean) - Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int) + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (empid > 15) (type: boolean) + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -665,7 +615,7 @@ POSTHOOK: query: drop materialized view mv1 POSTHOOK: type: DROP_MATERIALIZED_VIEW POSTHOOK: Input: default@mv1 POSTHOOK: Output: default@mv1 -Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join MERGEJOIN[18][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product PREHOOK: query: create materialized view mv1 enable rewrite as select depts.name, dependents.name as name2, emps.deptno, depts.deptno as deptno2, dependents.empid from depts, dependents, emps @@ -720,41 +670,51 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 8 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((deptno = deptno2) and (name = name2)) (type: boolean) - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int) + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 8 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((deptno = deptno2) and (name = name2)) (type: boolean) + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_3.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_3.q.out similarity index 72% rename from ql/src/test/results/clientpositive/materialized_view_rewrite_3.q.out rename to ql/src/test/results/clientpositive/llap/materialized_view_rewrite_3.q.out index e3bd233177..353a5d8acb 100644 --- a/ql/src/test/results/clientpositive/materialized_view_rewrite_3.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_3.q.out @@ -207,36 +207,21 @@ select empid deptno from emps join depts using (deptno) where empid = 1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (deptno = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: default.mv1 + Filter Operator + predicate: (deptno = 1) (type: boolean) + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + ListSink PREHOOK: query: select empid deptno from emps join depts using (deptno) where empid = 1 @@ -295,36 +280,21 @@ select empid deptno from emps join depts using (deptno) where empid > 1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToInteger(_c0) > 1) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: UDFToInteger(_c0) (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: default.mv1 + Filter Operator + predicate: (UDFToInteger(_c0) > 1) (type: boolean) + Select Operator + expressions: UDFToInteger(_c0) (type: int) + outputColumnNames: _col0 + ListSink PREHOOK: query: select empid deptno from emps join depts using (deptno) where empid > 1 @@ -387,36 +357,21 @@ select empid deptno from emps join depts using (deptno) where empid = 1 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToInteger(_c0) = 1) (type: boolean) - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: 1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: default.mv1 + Filter Operator + predicate: (UDFToInteger(_c0) = 1) (type: boolean) + Select Operator + expressions: 1 (type: int) + outputColumnNames: _col0 + ListSink PREHOOK: query: select empid deptno from emps join depts using (deptno) where empid = 1 @@ -486,59 +441,73 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 4 Data size: 352 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: name is not null (type: boolean) - Statistics: Num rows: 4 Data size: 352 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: name (type: varchar(256)) - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 352 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col0 (type: varchar(256)) + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 Statistics: Num rows: 4 Data size: 352 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: dependents - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: name is not null (type: boolean) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int), name (type: varchar(256)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col1 (type: varchar(256)) + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 4 Data size: 352 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 352 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 4 Data size: 352 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Map 3 + Map Operator Tree: + TableScan + alias: dependents Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: varchar(256)) - 1 _col1 (type: varchar(256)) - outputColumnNames: _col1 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: varchar(256)) + 1 _col1 (type: varchar(256)) + outputColumnNames: _col1 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_4.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_4.q.out similarity index 60% rename from ql/src/test/results/clientpositive/materialized_view_rewrite_4.q.out rename to ql/src/test/results/clientpositive/llap/materialized_view_rewrite_4.q.out index 7301571cbf..4da2ed316d 100644 --- a/ql/src/test/results/clientpositive/materialized_view_rewrite_4.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_4.q.out @@ -210,41 +210,51 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: name (type: varchar(256)), c (type: bigint), s (type: bigint) - outputColumnNames: name, c, s - Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)), c (type: bigint), s (type: bigint) + outputColumnNames: name, c, s + Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: $sum0(c), sum(s) + keys: name (type: varchar(256)) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint), _col2 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - aggregations: $sum0(c), sum(s) - keys: name (type: varchar(256)) - mode: hash + aggregations: $sum0(VALUE._col0), sum(VALUE._col1) + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col0 (type: varchar(256)) + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: $sum0(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: varchar(256)) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -318,11 +328,9 @@ STAGE PLANS: Processor Tree: TableScan alias: default.mv1 - Statistics: Num rows: 4 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: salary (type: float), name (type: varchar(256)), s (type: bigint), c (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select salary, name, sum(empid) as s, count(*) as c @@ -393,41 +401,51 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 4 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int), c (type: bigint), s (type: bigint) - outputColumnNames: deptno, c, s - Statistics: Num rows: 4 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), c (type: bigint), s (type: bigint) + outputColumnNames: deptno, c, s + Statistics: Num rows: 4 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: $sum0(c), sum(s) + keys: deptno (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint), _col2 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - aggregations: $sum0(c), sum(s) - keys: deptno (type: int) - mode: hash + aggregations: $sum0(VALUE._col0), sum(VALUE._col1) + keys: KEY._col0 (type: int) + mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint), _col2 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: $sum0(VALUE._col0), sum(VALUE._col1) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -505,40 +523,50 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (deptno > 10) (type: boolean) - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno > 10) (type: boolean) + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(s) + keys: deptno (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - aggregations: sum(s) - keys: deptno (type: int) - mode: hash + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -615,48 +643,58 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (deptno > 10) (type: boolean) - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int), s (type: bigint) + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno > 10) (type: boolean) + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), s (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col1) + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(_col1) - keys: _col0 (type: int) - mode: hash + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), (_col1 + 1L) (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), (_col1 + 1L) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -735,91 +773,96 @@ group by dependents.empid POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: name is not null (type: boolean) - Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: name (type: varchar(256)), s (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col0 (type: varchar(256)) + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) - TableScan - alias: dependents - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: name is not null (type: boolean) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int), name (type: varchar(256)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col1 (type: varchar(256)) + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)), s (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Execution mode: llap + LLAP IO: all inputs + Map 4 + Map Operator Tree: + TableScan + alias: dependents Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: varchar(256)) - 1 _col1 (type: varchar(256)) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(_col1) - keys: _col2 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: varchar(256)) + 1 _col1 (type: varchar(256)) + outputColumnNames: _col1, _col2 + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col1) + keys: _col2 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -900,21 +943,107 @@ join dependents on (emps.empid = dependents.empid) group by dependents.empid, emps.deptno POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: emps + Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), deptno (type: int), salary (type: float) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: int), _col2 (type: float) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: dependents + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col1 (type: int), _col3 (type: int), _col2 (type: float) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: float) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: float) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col2 (type: float), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(_col1) + keys: _col0 (type: int), _col2 (type: int) + mode: complete + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col2 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int), s (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - ListSink + ListSink PREHOOK: query: select emps.deptno, count(distinct salary) as s from emps @@ -923,7 +1052,6 @@ group by dependents.empid, emps.deptno PREHOOK: type: QUERY PREHOOK: Input: default@dependents PREHOOK: Input: default@emps -PREHOOK: Input: default@mv1 #### A masked pattern was here #### POSTHOOK: query: select emps.deptno, count(distinct salary) as s from emps @@ -932,7 +1060,6 @@ group by dependents.empid, emps.deptno POSTHOOK: type: QUERY POSTHOOK: Input: default@dependents POSTHOOK: Input: default@emps -POSTHOOK: Input: default@mv1 #### A masked pattern was here #### PREHOOK: query: drop materialized view mv1 PREHOOK: type: DROP_MATERIALIZED_VIEW diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_5.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_5.q.out similarity index 72% rename from ql/src/test/results/clientpositive/materialized_view_rewrite_5.q.out rename to ql/src/test/results/clientpositive/llap/materialized_view_rewrite_5.q.out index bab3cacc2c..ceb5900d4b 100644 --- a/ql/src/test/results/clientpositive/materialized_view_rewrite_5.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_5.q.out @@ -222,41 +222,51 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (deptno >= 20) (type: boolean) - Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: name (type: varchar(256)) - outputColumnNames: name - Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: name (type: varchar(256)) - mode: hash - outputColumnNames: _col0 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno >= 20) (type: boolean) + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: name + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: name (type: varchar(256)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col0 (type: varchar(256)) - Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: varchar(256)) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -321,44 +331,54 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (deptno > 15) (type: boolean) - Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: name (type: varchar(256)), s (type: bigint) - outputColumnNames: name, s - Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(s) - keys: name (type: varchar(256)) - mode: hash - outputColumnNames: _col0, _col1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno > 15) (type: boolean) + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)), s (type: bigint) + outputColumnNames: name, s + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(s) + keys: name (type: varchar(256)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col0 (type: varchar(256)) - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: varchar(256)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -451,41 +471,51 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((deptno < 19) and (deptno > 11)) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int) + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((deptno < 19) and (deptno > 11)) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -569,38 +599,48 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - keys: _col0 (type: int) - mode: hash + keys: KEY._col0 (type: int) + mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -674,11 +714,9 @@ STAGE PLANS: Processor Tree: TableScan alias: default.mv1 - Statistics: Num rows: 4 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: deptno (type: int), empid (type: int), s (type: bigint), c (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select deptno, empid, sum(empid) as s, count(*) as c @@ -751,84 +789,89 @@ group by dependents.empid POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: depts - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: default.mv1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int), deptno (type: int), s (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: double) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col1 (type: int) - outputColumnNames: _col1, _col3 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(_col3) - keys: _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), deptno (type: int), s (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col1 (type: int) + outputColumnNames: _col1, _col3 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col3) + keys: _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -912,85 +955,90 @@ group by depts.name POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: depts - Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int), name (type: varchar(256)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: varchar(256)) - TableScan - alias: default.mv1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int), s (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col1, _col3 - Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(_col3) - keys: _col1 (type: varchar(256)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col0 (type: varchar(256)) - Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: varchar(256)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: varchar(256)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), s (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1, _col3 + Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col3) + keys: _col1 (type: varchar(256)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 97 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1028,7 +1076,7 @@ POSTHOOK: query: drop materialized view mv1 POSTHOOK: type: DROP_MATERIALIZED_VIEW POSTHOOK: Input: default@mv1 POSTHOOK: Output: default@mv1 -Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 3' is a cross product PREHOOK: query: create materialized view mv1 enable rewrite as select a.empid deptno from (select * from emps where empid = 1) a @@ -1081,11 +1129,9 @@ STAGE PLANS: Processor Tree: TableScan alias: default.mv1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Select Operator expressions: 1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE ListSink PREHOOK: query: select a.empid from @@ -1112,7 +1158,7 @@ POSTHOOK: query: drop materialized view mv1 POSTHOOK: type: DROP_MATERIALIZED_VIEW POSTHOOK: Input: default@mv1 POSTHOOK: Output: default@mv1 -Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 3' is a cross product PREHOOK: query: create materialized view mv1 enable rewrite as select a.empid, a.deptno from (select * from emps where empid = 1) a @@ -1165,11 +1211,9 @@ STAGE PLANS: Processor Tree: TableScan alias: default.mv1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Select Operator expressions: 1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE ListSink PREHOOK: query: select a.empid from @@ -1240,11 +1284,9 @@ STAGE PLANS: Processor Tree: TableScan alias: default.mv1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Select Operator expressions: 1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE ListSink PREHOOK: query: select empid from emps where empid = 1 @@ -1265,7 +1307,7 @@ POSTHOOK: query: drop materialized view mv1 POSTHOOK: type: DROP_MATERIALIZED_VIEW POSTHOOK: Input: default@mv1 POSTHOOK: Output: default@mv1 -Warning: Shuffle Join JOIN[13][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 3' is a cross product PREHOOK: query: create materialized view mv1 enable rewrite as select emps.empid, emps.deptno from emps join depts on (emps.deptno = depts.deptno) @@ -1318,11 +1360,9 @@ STAGE PLANS: Processor Tree: TableScan alias: default.mv1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Select Operator expressions: 1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE ListSink PREHOOK: query: select emps.empid from emps @@ -1349,7 +1389,7 @@ POSTHOOK: query: drop materialized view mv1 POSTHOOK: type: DROP_MATERIALIZED_VIEW POSTHOOK: Input: default@mv1 POSTHOOK: Output: default@mv1 -Warning: Shuffle Join JOIN[16][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_0]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_0]] in Stage 'Reducer 3' is a cross product PREHOOK: query: create materialized view mv1 enable rewrite as select emps.empid, emps.deptno from emps join depts a on (emps.deptno=a.deptno) @@ -1404,11 +1444,9 @@ STAGE PLANS: Processor Tree: TableScan alias: default.mv1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Select Operator expressions: 1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE ListSink PREHOOK: query: select emps.empid from emps @@ -1489,11 +1527,9 @@ STAGE PLANS: Processor Tree: TableScan alias: default.mv1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: empid (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: select emps.empid from emps diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_6.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_6.q.out similarity index 58% rename from ql/src/test/results/clientpositive/materialized_view_rewrite_6.q.out rename to ql/src/test/results/clientpositive/llap/materialized_view_rewrite_6.q.out index 017d793283..80c21eda9f 100644 --- a/ql/src/test/results/clientpositive/materialized_view_rewrite_6.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_6.q.out @@ -226,48 +226,58 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (deptno > 10) (type: boolean) - Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: salary (type: float), s (type: bigint) - outputColumnNames: salary, s - Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(s) - keys: salary (type: float) - mode: hash + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno > 10) (type: boolean) + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: salary (type: float), s (type: bigint) + outputColumnNames: salary, s + Statistics: Num rows: 4 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(s) + keys: salary (type: float) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: float) + sort order: + + Map-reduce partition columns: _col0 (type: float) + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: float) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: float), (_col1 + 1L) (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: float) - sort order: + - Map-reduce partition columns: _col0 (type: float) + File Output Operator + compressed: false Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: float) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: float), (_col1 + 1L) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -334,48 +344,58 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (deptno > 15) (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: salary (type: float), s (type: bigint) - outputColumnNames: salary, s - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(s) - keys: salary (type: float) - mode: hash + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (deptno > 15) (type: boolean) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: salary (type: float), s (type: bigint) + outputColumnNames: salary, s + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(s) + keys: salary (type: float) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: float) + sort order: + + Map-reduce partition columns: _col0 (type: float) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: float) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: (_col0 + 1.0) (type: float), (_col1 + 1L) (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: float) - sort order: + - Map-reduce partition columns: _col0 (type: float) + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: float) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (_col0 + 1.0) (type: float), (_col1 + 1L) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -450,59 +470,73 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: name is not null (type: boolean) - Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: name (type: varchar(256)) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col0 (type: varchar(256)) + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: dependents - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: name is not null (type: boolean) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int), name (type: varchar(256)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col1 (type: varchar(256)) + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Map 3 + Map Operator Tree: + TableScan + alias: dependents Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: varchar(256)) - 1 _col1 (type: varchar(256)) - outputColumnNames: _col1 - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: varchar(256)) + 1 _col1 (type: varchar(256)) + outputColumnNames: _col1 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -586,76 +620,94 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: name is not null (type: boolean) - Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: name (type: varchar(256)) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col0 (type: varchar(256)) + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1 Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: locations - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: name is not null (type: boolean) - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: name (type: varchar(256)) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col0 (type: varchar(256)) + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 5 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Map 3 + Map Operator Tree: + TableScan + alias: locations Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: dependents - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: name is not null (type: boolean) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int), name (type: varchar(256)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col1 (type: varchar(256)) + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: dependents Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 _col0 (type: varchar(256)) - 1 _col0 (type: varchar(256)) - 2 _col1 (type: varchar(256)) - outputColumnNames: _col2 - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col0 (type: varchar(256)) + 1 _col0 (type: varchar(256)) + 2 _col1 (type: varchar(256)) + outputColumnNames: _col2 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -741,56 +793,70 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: default.mv1 - Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (UDFToString(name1) = 'Bill') (type: boolean) - Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int), deptno (type: int) + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 3 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToString(name1) = 'Bill') (type: boolean) + Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), deptno (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col1 (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col1 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: int), _col1 (type: int), _col0 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col1 (type: int), _col1 (type: int), _col0 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_7.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_7.q.out new file mode 100644 index 0000000000..a199902f0b --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_7.q.out @@ -0,0 +1,1022 @@ +PREHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@emps +POSTHOOK: query: create table emps ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@emps +PREHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@emps +POSTHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@emps +POSTHOOK: Lineage: emps.commission SCRIPT [] +POSTHOOK: Lineage: emps.deptno SCRIPT [] +POSTHOOK: Lineage: emps.empid SCRIPT [] +POSTHOOK: Lineage: emps.name SCRIPT [] +POSTHOOK: Lineage: emps.salary SCRIPT [] +PREHOOK: query: analyze table emps compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@emps +PREHOOK: Output: default@emps +#### A masked pattern was here #### +POSTHOOK: query: analyze table emps compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps +POSTHOOK: Output: default@emps +#### A masked pattern was here #### +PREHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@depts +POSTHOOK: query: create table depts ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@depts +PREHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@depts +POSTHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@depts +POSTHOOK: Lineage: depts.deptno SCRIPT [] +POSTHOOK: Lineage: depts.locationid SCRIPT [] +POSTHOOK: Lineage: depts.name SCRIPT [] +PREHOOK: query: analyze table depts compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@depts +PREHOOK: Output: default@depts +#### A masked pattern was here #### +POSTHOOK: query: analyze table depts compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts +POSTHOOK: Output: default@depts +#### A masked pattern was here #### +PREHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dependents +POSTHOOK: query: create table dependents ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dependents +PREHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dependents +POSTHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dependents +POSTHOOK: Lineage: dependents.empid SCRIPT [] +POSTHOOK: Lineage: dependents.name SCRIPT [] +PREHOOK: query: analyze table dependents compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Output: default@dependents +#### A masked pattern was here #### +POSTHOOK: query: analyze table dependents compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Output: default@dependents +#### A masked pattern was here #### +PREHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@locations +POSTHOOK: query: create table locations ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@locations +PREHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@locations +POSTHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@locations +POSTHOOK: Lineage: locations.locationid SCRIPT [] +POSTHOOK: Lineage: locations.name SCRIPT [] +PREHOOK: query: analyze table locations compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@locations +PREHOOK: Output: default@locations +#### A masked pattern was here #### +POSTHOOK: query: analyze table locations compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@locations +POSTHOOK: Output: default@locations +#### A masked pattern was here #### +PREHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table emps change column deptno deptno int constraint nn1 not null disable novalidate rely +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@emps +PREHOOK: Output: default@emps +POSTHOOK: query: alter table emps change column deptno deptno int constraint nn1 not null disable novalidate rely +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@emps +POSTHOOK: Output: default@emps +PREHOOK: query: alter table depts change column locationid locationid int constraint nn2 not null disable novalidate rely +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@depts +PREHOOK: Output: default@depts +POSTHOOK: query: alter table depts change column locationid locationid int constraint nn2 not null disable novalidate rely +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@depts +POSTHOOK: Output: default@depts +PREHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 +group by depts.deptno, dependents.empid +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@locations +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 +group by depts.deptno, dependents.empid +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@locations +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select dependents.empid, depts.deptno +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid, depts.deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select dependents.empid, depts.deptno +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid, depts.deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 10 <- Union 4 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE) + Reducer 8 <- Map 7 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: dependents + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 10 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), deptno (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Map 6 + Map Operator Tree: + TableScan + alias: locations + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 7 + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((deptno <= 11) and (deptno > 10) and name is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: varchar(256)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 9 + Map Operator Tree: + TableScan + alias: emps + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((deptno <= 11) and (deptno > 10)) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col1 (type: varchar(256)) + 1 _col0 (type: varchar(256)) + 2 _col1 (type: varchar(256)) + outputColumnNames: _col0, _col3 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col3 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col1 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 8 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Union 4 + Vertex: Union 4 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select dependents.empid, depts.deptno +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid, depts.deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@locations +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select dependents.empid, depts.deptno +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 +group by dependents.empid, depts.deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@locations +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid, count(emps.salary) as s +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 and depts.deptno < 19 +group by depts.deptno, dependents.empid +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@locations +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid, count(emps.salary) as s +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 11 and depts.deptno < 19 +group by depts.deptno, dependents.empid +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@locations +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select dependents.empid, count(emps.salary) + 1 +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 and depts.deptno < 20 +group by dependents.empid +PREHOOK: type: QUERY +POSTHOOK: query: explain +select dependents.empid, count(emps.salary) + 1 +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 and depts.deptno < 20 +group by dependents.empid +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 11 <- Map 10 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE) + Reducer 8 <- Map 7 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: dependents + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 10 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), s (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: $sum0(_col1) + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Map 6 + Map Operator Tree: + TableScan + alias: locations + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: name (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 7 + Map Operator Tree: + TableScan + alias: emps + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (((deptno <= 11) or (deptno >= 19)) and (deptno < 20) and (deptno > 10)) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), salary (type: float) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: float) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 9 + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (((deptno <= 11) or (deptno >= 19)) and (deptno < 20) and (deptno > 10) and name is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: varchar(256)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 11 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: $sum0(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: $sum0(_col1) + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col1 (type: varchar(256)) + 1 _col0 (type: varchar(256)) + 2 _col3 (type: varchar(256)) + outputColumnNames: _col0, _col4 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(_col4) + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: $sum0(_col1) + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: $sum0(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), (_col1 + 1L) (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 8 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1, _col3 + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col3 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col3 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: float) + Union 4 + Vertex: Union 4 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select dependents.empid, count(emps.salary) + 1 +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 and depts.deptno < 20 +group by dependents.empid +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@locations +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select dependents.empid, count(emps.salary) + 1 +from depts +join dependents on (depts.name = dependents.name) +join locations on (locations.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 10 and depts.deptno < 20 +group by dependents.empid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@locations +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +PREHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno >= 10 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1 +POSTHOOK: query: create materialized view mv1 enable rewrite as +select depts.deptno, dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno >= 10 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1 +PREHOOK: query: analyze table mv1 compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1 compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 0 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 0 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 7 <- Union 4 (CONTAINS) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: depts + Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((deptno < 10) and (deptno > 0) and name is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: varchar(256)) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 5 + Map Operator Tree: + TableScan + alias: emps + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((deptno < 10) and (deptno > 0)) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 6 + Map Operator Tree: + TableScan + alias: dependents + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), name (type: varchar(256)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 7 + Map Operator Tree: + TableScan + alias: default.mv1 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1 + Statistics: Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col1 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: varchar(256)) + 1 _col1 (type: varchar(256)) + outputColumnNames: _col3 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col3 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 4 + Vertex: Union 4 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents +PREHOOK: Input: default@depts +PREHOOK: Input: default@emps +PREHOOK: Input: default@mv1 +#### A masked pattern was here #### +POSTHOOK: query: select dependents.empid +from depts +join dependents on (depts.name = dependents.name) +join emps on (emps.deptno = depts.deptno) +where depts.deptno > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents +POSTHOOK: Input: default@depts +POSTHOOK: Input: default@emps +POSTHOOK: Input: default@mv1 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1 +PREHOOK: Output: default@mv1 +POSTHOOK: query: drop materialized view mv1 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1 +POSTHOOK: Output: default@mv1 diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_8.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_8.q.out similarity index 52% rename from ql/src/test/results/clientpositive/materialized_view_rewrite_8.q.out rename to ql/src/test/results/clientpositive/llap/materialized_view_rewrite_8.q.out index 1ca06d3cc4..3dcbd5575e 100644 --- a/ql/src/test/results/clientpositive/materialized_view_rewrite_8.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_8.q.out @@ -106,11 +106,9 @@ STAGE PLANS: Processor Tree: TableScan alias: default.source_table_001_mv - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: down_volume_sum (type: bigint), my_date (type: date), my_id2 (type: bigint), environment (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: explain @@ -139,14 +137,11 @@ STAGE PLANS: Processor Tree: TableScan alias: default.source_table_001_mv - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: down_volume_sum (type: bigint), my_date (type: date), my_id2 (type: bigint), environment (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 100 - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: explain @@ -175,11 +170,9 @@ STAGE PLANS: Processor Tree: TableScan alias: default.source_table_001_mv - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: 1 (type: int), down_volume_sum (type: bigint), my_date (type: date), my_id2 (type: bigint), environment (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 163 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: explain @@ -197,33 +190,19 @@ FROM source_table_001 AS A group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.source_table_001_mv - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (down_volume_sum + 0L) (type: bigint), my_date (type: date), my_id2 (type: bigint), environment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: default.source_table_001_mv + Select Operator + expressions: (down_volume_sum + 0L) (type: bigint), my_date (type: date), my_id2 (type: bigint), environment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + ListSink PREHOOK: query: explain select @@ -249,36 +228,46 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.source_table_001_mv - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: down_volume_sum (type: bigint), my_date (type: date), my_id2 (type: bigint), environment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: bigint) - sort order: + + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint), my_date (type: date), my_id2 (type: bigint), environment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col2 (type: bigint) + sort order: + + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: bigint), _col1 (type: date), _col3 (type: string) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: bigint), VALUE._col1 (type: date), KEY.reducesinkkey0 (type: bigint), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col0 (type: bigint), _col1 (type: date), _col3 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: bigint), VALUE._col1 (type: date), KEY.reducesinkkey0 (type: bigint), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 100 - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Limit + Number of rows: 100 + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -302,38 +291,48 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.source_table_001_mv - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: my_date (type: date), my_id2 (type: bigint), environment (type: string) - outputColumnNames: my_date, my_id2, environment - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: my_date (type: date), my_id2 (type: bigint), environment (type: string) + outputColumnNames: my_date, my_id2, environment + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: my_date (type: date), my_id2 (type: bigint), environment (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: date), _col1 (type: bigint), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: date), _col1 (type: bigint), _col2 (type: string) + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - keys: my_date (type: date), my_id2 (type: bigint), environment (type: string) - mode: hash + keys: KEY._col0 (type: date), KEY._col1 (type: bigint), KEY._col2 (type: string) + mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: date), _col1 (type: bigint), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: date), _col1 (type: bigint), _col2 (type: string) + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: date), KEY._col1 (type: bigint), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -358,36 +357,21 @@ where A.MY_DATE=TO_DATE('2010-01-10') group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.source_table_001_mv - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (my_date = DATE'2010-01-10') (type: boolean) - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: down_volume_sum (type: bigint), DATE'2010-01-10' (type: date), my_id2 (type: bigint), environment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: default.source_table_001_mv + Filter Operator + predicate: (my_date = DATE'2010-01-10') (type: boolean) + Select Operator + expressions: down_volume_sum (type: bigint), DATE'2010-01-10' (type: date), my_id2 (type: bigint), environment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + ListSink PREHOOK: query: explain select @@ -406,36 +390,21 @@ where A.MY_DATE=TO_DATE('2010-01-10') group by A.MY_ID,A.MY_ID2,A.ENVIRONMENT,A.MY_DATE POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.source_table_001_mv - Statistics: Num rows: 1 Data size: 167 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (my_date = DATE'2010-01-10') (type: boolean) - Statistics: Num rows: 1 Data size: 167 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: (down_volume_sum + up_volume_sum) (type: bigint), DATE'2010-01-10' (type: date), my_id2 (type: bigint), environment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: default.source_table_001_mv + Filter Operator + predicate: (my_date = DATE'2010-01-10') (type: boolean) + Select Operator + expressions: (down_volume_sum + up_volume_sum) (type: bigint), DATE'2010-01-10' (type: date), my_id2 (type: bigint), environment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + ListSink PREHOOK: query: explain select @@ -455,40 +424,50 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.source_table_001_mv - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (my_date = DATE'2010-01-10') (type: boolean) - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: down_volume_sum (type: bigint) - outputColumnNames: down_volume_sum - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(down_volume_sum) - mode: hash - outputColumnNames: _col0 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (my_date = DATE'2010-01-10') (type: boolean) + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint) + outputColumnNames: down_volume_sum + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(down_volume_sum) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -516,44 +495,54 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.source_table_001_mv - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (my_date = DATE'2010-01-10') (type: boolean) - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: down_volume_sum (type: bigint) - outputColumnNames: down_volume_sum - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(down_volume_sum) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: bigint), DATE'2010-01-10' (type: date) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (my_date = DATE'2010-01-10') (type: boolean) + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint) + outputColumnNames: down_volume_sum + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(down_volume_sum) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint), DATE'2010-01-10' (type: date) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -583,48 +572,58 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.source_table_001_mv - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (my_date = DATE'2010-01-10') (type: boolean) - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: down_volume_sum (type: bigint) - outputColumnNames: _col0 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (my_date = DATE'2010-01-10') (type: boolean) + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + keys: DATE'2010-01-10' (type: date) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: date) + sort order: + + Map-reduce partition columns: _col0 (type: date) + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: date) + mode: mergepartial + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: sum(_col0) - keys: DATE'2010-01-10' (type: date) - mode: hash + Select Operator + expressions: _col1 (type: bigint), DATE'2010-01-10' (type: date) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: date) - sort order: + - Map-reduce partition columns: _col0 (type: date) + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: date) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: bigint), DATE'2010-01-10' (type: date) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_9.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_9.q.out similarity index 52% rename from ql/src/test/results/clientpositive/materialized_view_rewrite_9.q.out rename to ql/src/test/results/clientpositive/llap/materialized_view_rewrite_9.q.out index 3120e0db7a..63daf6ccb0 100644 --- a/ql/src/test/results/clientpositive/materialized_view_rewrite_9.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_9.q.out @@ -102,45 +102,55 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.source_table_001_mv - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: down_volume_sum (type: bigint), my_id (type: bigint), my_id2 (type: bigint), environment (type: string), floor_hour(my_date) (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint), my_id (type: bigint), my_id2 (type: bigint), environment (type: string), floor_hour(my_date) (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + keys: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: string), _col4 (type: timestamp) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) + sort order: ++++ + Map-reduce partition columns: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col4 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - aggregations: sum(_col0) - keys: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: string), _col4 (type: timestamp) - mode: hash + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: bigint), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: timestamp) + mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) - sort order: ++++ - Map-reduce partition columns: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col4 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: bigint), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: timestamp) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col4 (type: bigint), _col3 (type: timestamp), _col1 (type: bigint), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col4 (type: bigint), _col3 (type: timestamp), _col1 (type: bigint), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -208,45 +218,55 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.source_table_001_mv - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: down_volume_sum (type: bigint), my_id (type: bigint), my_id2 (type: bigint), environment (type: string), floor_day(_c3) (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.source_table_001_mv + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: down_volume_sum (type: bigint), my_id (type: bigint), my_id2 (type: bigint), environment (type: string), floor_day(_c3) (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col0) + keys: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: string), _col4 (type: timestamp) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) + sort order: ++++ + Map-reduce partition columns: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col4 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - aggregations: sum(_col0) - keys: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: string), _col4 (type: timestamp) - mode: hash + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: bigint), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: timestamp) + mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) - sort order: ++++ - Map-reduce partition columns: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col4 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: bigint), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: timestamp) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col4 (type: bigint), _col3 (type: timestamp), _col1 (type: bigint), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col4 (type: bigint), _col3 (type: timestamp), _col1 (type: bigint), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -278,11 +298,9 @@ STAGE PLANS: Processor Tree: TableScan alias: default.source_table_001_mv - Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: down_volume_sum (type: bigint), _c3 (type: timestamp), my_id2 (type: bigint), environment (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: explain @@ -305,45 +323,55 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: a - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: my_id (type: bigint), my_id2 (type: bigint), environment (type: string), floor_second(my_date) (type: timestamp), down_volume (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: my_id (type: bigint), my_id2 (type: bigint), environment (type: string), floor_second(my_date) (type: timestamp), down_volume (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(_col4) + keys: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) + sort order: ++++ + Map-reduce partition columns: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) + Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col4 (type: bigint) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: Group By Operator - aggregations: sum(_col4) - keys: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) - mode: hash + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: bigint), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: timestamp) + mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) - sort order: ++++ - Map-reduce partition columns: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: string), _col3 (type: timestamp) - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col4 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: bigint), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: timestamp) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 151 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col4 (type: bigint), _col3 (type: timestamp), _col1 (type: bigint), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: _col4 (type: bigint), _col3 (type: timestamp), _col1 (type: bigint), _col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 143 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out b/ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out deleted file mode 100644 index 18eb1d1daa..0000000000 --- a/ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out +++ /dev/null @@ -1,477 +0,0 @@ -PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_basetable -POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_basetable -PREHOOK: query: insert into cmv_basetable values - (1, 'alfred', 10.30, 2), - (2, 'bob', 3.14, 3), - (2, 'bonnie', 172342.2, 3), - (3, 'calvin', 978.76, 3), - (3, 'charlie', 9.8, 1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@cmv_basetable -POSTHOOK: query: insert into cmv_basetable values - (1, 'alfred', 10.30, 2), - (2, 'bob', 3.14, 3), - (2, 'bonnie', 172342.2, 3), - (3, 'calvin', 978.76, 3), - (3, 'charlie', 9.8, 1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@cmv_basetable -POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] -POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] -POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] -POSTHOOK: Lineage: cmv_basetable.d SCRIPT [] -PREHOOK: query: analyze table cmv_basetable compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Output: default@cmv_basetable -#### A masked pattern was here #### -POSTHOOK: query: analyze table cmv_basetable compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Output: default@cmv_basetable -#### A masked pattern was here #### -PREHOOK: query: create materialized view cmv_mat_view enable rewrite -as select a, b, c from cmv_basetable where a = 2 -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_basetable -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_mat_view -POSTHOOK: query: create materialized view cmv_mat_view enable rewrite -as select a, b, c from cmv_basetable where a = 2 -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_mat_view -PREHOOK: query: select * from cmv_mat_view -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_mat_view -#### A masked pattern was here #### -POSTHOOK: query: select * from cmv_mat_view -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_mat_view -#### A masked pattern was here #### -2 bob 3.14 -2 bonnie 172342.20 -PREHOOK: query: show tblproperties cmv_mat_view -PREHOOK: type: SHOW_TBLPROPERTIES -POSTHOOK: query: show tblproperties cmv_mat_view -POSTHOOK: type: SHOW_TBLPROPERTIES -COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} -numFiles 1 -numRows 2 -rawDataSize 408 -totalSize 453 -#### A masked pattern was here #### -PREHOOK: query: create materialized view if not exists cmv_mat_view2 enable rewrite -as select a, c from cmv_basetable where a = 3 -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_basetable -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_mat_view2 -POSTHOOK: query: create materialized view if not exists cmv_mat_view2 enable rewrite -as select a, c from cmv_basetable where a = 3 -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_mat_view2 -PREHOOK: query: select * from cmv_mat_view2 -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_mat_view2 -#### A masked pattern was here #### -POSTHOOK: query: select * from cmv_mat_view2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_mat_view2 -#### A masked pattern was here #### -3 9.80 -3 978.76 -PREHOOK: query: show tblproperties cmv_mat_view2 -PREHOOK: type: SHOW_TBLPROPERTIES -POSTHOOK: query: show tblproperties cmv_mat_view2 -POSTHOOK: type: SHOW_TBLPROPERTIES -COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} -numFiles 1 -numRows 2 -rawDataSize 232 -totalSize 322 -#### A masked pattern was here #### -PREHOOK: query: explain -select a, c from cmv_basetable where a = 3 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select a, c from cmv_basetable where a = 3 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: default.cmv_mat_view2 - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: 3 (type: int), c (type: decimal(10,2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE - ListSink - -PREHOOK: query: select a, c from cmv_basetable where a = 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_mat_view2 -#### A masked pattern was here #### -POSTHOOK: query: select a, c from cmv_basetable where a = 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_mat_view2 -#### A masked pattern was here #### -3 9.80 -3 978.76 -PREHOOK: query: alter materialized view cmv_mat_view2 disable rewrite -PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE -PREHOOK: Input: default@cmv_mat_view2 -PREHOOK: Output: default@cmv_mat_view2 -POSTHOOK: query: alter materialized view cmv_mat_view2 disable rewrite -POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE -POSTHOOK: Input: default@cmv_mat_view2 -POSTHOOK: Output: default@cmv_mat_view2 -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -PREHOOK: query: explain -select * from ( - (select a, c from cmv_basetable where a = 3) table1 - join - (select a, c from cmv_basetable where d = 3) table2 - on table1.a = table2.a) -PREHOOK: type: QUERY -POSTHOOK: query: explain -select * from ( - (select a, c from cmv_basetable where a = 3) table1 - join - (select a, c from cmv_basetable where d = 3) table2 - on table1.a = table2.a) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (a = 3) (type: boolean) - Statistics: Num rows: 2 Data size: 482 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c (type: decimal(10,2)) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 482 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 2 Data size: 482 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: decimal(10,2)) - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((a = 3) and (d = 3)) (type: boolean) - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c (type: decimal(10,2)) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: decimal(10,2)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: 3 (type: int), _col0 (type: decimal(10,2)), 3 (type: int), _col1 (type: decimal(10,2)) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -PREHOOK: query: select * from ( - (select a, c from cmv_basetable where a = 3) table1 - join - (select a, c from cmv_basetable where d = 3) table2 - on table1.a = table2.a) -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -#### A masked pattern was here #### -POSTHOOK: query: select * from ( - (select a, c from cmv_basetable where a = 3) table1 - join - (select a, c from cmv_basetable where d = 3) table2 - on table1.a = table2.a) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -#### A masked pattern was here #### -3 9.80 3 978.76 -3 978.76 3 978.76 -PREHOOK: query: explain -alter materialized view cmv_mat_view2 enable rewrite -PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE -POSTHOOK: query: explain -alter materialized view cmv_mat_view2 enable rewrite -POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Alter Materialized View Operator: - Alter Materialized View - name: default.cmv_mat_view2 - operation: UPDATE_REWRITE_FLAG - -PREHOOK: query: alter materialized view cmv_mat_view2 enable rewrite -PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE -PREHOOK: Input: default@cmv_mat_view2 -PREHOOK: Output: default@cmv_mat_view2 -POSTHOOK: query: alter materialized view cmv_mat_view2 enable rewrite -POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE -POSTHOOK: Input: default@cmv_mat_view2 -POSTHOOK: Output: default@cmv_mat_view2 -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -PREHOOK: query: explain -select * from ( - (select a, c from cmv_basetable where a = 3) table1 - join - (select a, c from cmv_basetable where d = 3) table2 - on table1.a = table2.a) -PREHOOK: type: QUERY -POSTHOOK: query: explain -select * from ( - (select a, c from cmv_basetable where a = 3) table1 - join - (select a, c from cmv_basetable where d = 3) table2 - on table1.a = table2.a) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.cmv_mat_view2 - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c (type: decimal(10,2)) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: decimal(10,2)) - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((a = 3) and (d = 3)) (type: boolean) - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c (type: decimal(10,2)) - outputColumnNames: _col1 - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(10,2)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col2 - Statistics: Num rows: 2 Data size: 716 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: 3 (type: int), _col0 (type: decimal(10,2)), 3 (type: int), _col2 (type: decimal(10,2)) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 716 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 716 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -PREHOOK: query: select * from ( - (select a, c from cmv_basetable where a = 3) table1 - join - (select a, c from cmv_basetable where d = 3) table2 - on table1.a = table2.a) -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_mat_view2 -#### A masked pattern was here #### -POSTHOOK: query: select * from ( - (select a, c from cmv_basetable where a = 3) table1 - join - (select a, c from cmv_basetable where d = 3) table2 - on table1.a = table2.a) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_mat_view2 -#### A masked pattern was here #### -3 9.80 3 978.76 -3 978.76 3 978.76 -PREHOOK: query: drop materialized view cmv_mat_view2 -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_mat_view2 -PREHOOK: Output: default@cmv_mat_view2 -POSTHOOK: query: drop materialized view cmv_mat_view2 -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_mat_view2 -POSTHOOK: Output: default@cmv_mat_view2 -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -PREHOOK: query: explain -select * from ( - (select a, c from cmv_basetable where a = 3) table1 - join - (select a, c from cmv_basetable where d = 3) table2 - on table1.a = table2.a) -PREHOOK: type: QUERY -POSTHOOK: query: explain -select * from ( - (select a, c from cmv_basetable where a = 3) table1 - join - (select a, c from cmv_basetable where d = 3) table2 - on table1.a = table2.a) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (a = 3) (type: boolean) - Statistics: Num rows: 2 Data size: 482 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c (type: decimal(10,2)) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 482 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 2 Data size: 482 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: decimal(10,2)) - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((a = 3) and (d = 3)) (type: boolean) - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: c (type: decimal(10,2)) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: decimal(10,2)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: 3 (type: int), _col0 (type: decimal(10,2)), 3 (type: int), _col1 (type: decimal(10,2)) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 966 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product -PREHOOK: query: select * from ( - (select a, c from cmv_basetable where a = 3) table1 - join - (select a, c from cmv_basetable where d = 3) table2 - on table1.a = table2.a) -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -#### A masked pattern was here #### -POSTHOOK: query: select * from ( - (select a, c from cmv_basetable where a = 3) table1 - join - (select a, c from cmv_basetable where d = 3) table2 - on table1.a = table2.a) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -#### A masked pattern was here #### -3 9.80 3 978.76 -3 978.76 3 978.76 -PREHOOK: query: drop materialized view cmv_mat_view -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_mat_view -PREHOOK: Output: default@cmv_mat_view -POSTHOOK: query: drop materialized view cmv_mat_view -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_mat_view -POSTHOOK: Output: default@cmv_mat_view diff --git a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_2.q.out b/ql/src/test/results/clientpositive/materialized_view_create_rewrite_2.q.out deleted file mode 100644 index 144f9d8420..0000000000 --- a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_2.q.out +++ /dev/null @@ -1,598 +0,0 @@ -PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_basetable -POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_basetable -PREHOOK: query: insert into cmv_basetable values - (1, 'alfred', 10.30, 2), - (2, 'bob', 3.14, 3), - (2, 'bonnie', 172342.2, 3), - (3, 'calvin', 978.76, 3), - (3, 'charlie', 9.8, 1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@cmv_basetable -POSTHOOK: query: insert into cmv_basetable values - (1, 'alfred', 10.30, 2), - (2, 'bob', 3.14, 3), - (2, 'bonnie', 172342.2, 3), - (3, 'calvin', 978.76, 3), - (3, 'charlie', 9.8, 1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@cmv_basetable -POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] -POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] -POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] -POSTHOOK: Lineage: cmv_basetable.d SCRIPT [] -PREHOOK: query: analyze table cmv_basetable compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Output: default@cmv_basetable -#### A masked pattern was here #### -POSTHOOK: query: analyze table cmv_basetable compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Output: default@cmv_basetable -#### A masked pattern was here #### -PREHOOK: query: create materialized view cmv_mat_view enable rewrite -as select b from cmv_basetable where c > 10.0 group by a, b, c -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_basetable -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_mat_view -POSTHOOK: query: create materialized view cmv_mat_view enable rewrite -as select b from cmv_basetable where c > 10.0 group by a, b, c -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_mat_view -PREHOOK: query: explain -select b from cmv_basetable where c > 20.0 group by a, b -PREHOOK: type: QUERY -POSTHOOK: query: explain -select b from cmv_basetable where c > 20.0 group by a, b -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (c > 20) (type: boolean) - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: varchar(256)) - outputColumnNames: a, b - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int), b (type: varchar(256)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: varchar(256)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: varchar(256)) - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: varchar(256)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: varchar(256)) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select b from cmv_basetable where c > 20.0 group by a, b -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -#### A masked pattern was here #### -POSTHOOK: query: select b from cmv_basetable where c > 20.0 group by a, b -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -#### A masked pattern was here #### -bonnie -calvin -PREHOOK: query: create materialized view cmv_mat_view_2 enable rewrite -as select b, c from cmv_basetable where c > 10.0 group by a, b, c -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_basetable -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_mat_view_2 -POSTHOOK: query: create materialized view cmv_mat_view_2 enable rewrite -as select b, c from cmv_basetable where c > 10.0 group by a, b, c -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_mat_view_2 -PREHOOK: query: explain -select b from cmv_basetable where c > 20.0 group by a, b -PREHOOK: type: QUERY -POSTHOOK: query: explain -select b from cmv_basetable where c > 20.0 group by a, b -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (c > 20) (type: boolean) - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: varchar(256)) - outputColumnNames: a, b - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int), b (type: varchar(256)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: varchar(256)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: varchar(256)) - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: varchar(256)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: varchar(256)) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select b from cmv_basetable where c > 20.0 group by a, b -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -#### A masked pattern was here #### -POSTHOOK: query: select b from cmv_basetable where c > 20.0 group by a, b -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -#### A masked pattern was here #### -bonnie -calvin -PREHOOK: query: create materialized view cmv_mat_view_3 enable rewrite -as select a, b, c from cmv_basetable where c > 10.0 group by a, b, c -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_basetable -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_mat_view_3 -POSTHOOK: query: create materialized view cmv_mat_view_3 enable rewrite -as select a, b, c from cmv_basetable where c > 10.0 group by a, b, c -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_mat_view_3 -PREHOOK: query: explain -select b from cmv_basetable where c > 20.0 group by a, b -PREHOOK: type: QUERY -POSTHOOK: query: explain -select b from cmv_basetable where c > 20.0 group by a, b -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.cmv_mat_view_3 - Statistics: Num rows: 3 Data size: 618 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (c > 20) (type: boolean) - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), b (type: varchar(256)) - outputColumnNames: a, b - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int), b (type: varchar(256)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: varchar(256)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: varchar(256)) - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: varchar(256)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: varchar(256)) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select b from cmv_basetable where c > 20.0 group by a, b -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_mat_view_3 -#### A masked pattern was here #### -POSTHOOK: query: select b from cmv_basetable where c > 20.0 group by a, b -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_mat_view_3 -#### A masked pattern was here #### -bonnie -calvin -PREHOOK: query: create materialized view cmv_mat_view_4 enable rewrite -as select a, b from cmv_basetable group by a, b -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_basetable -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_mat_view_4 -POSTHOOK: query: create materialized view cmv_mat_view_4 enable rewrite -as select a, b from cmv_basetable group by a, b -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_mat_view_4 -PREHOOK: query: explain -select b from cmv_basetable group by b -PREHOOK: type: QUERY -POSTHOOK: query: explain -select b from cmv_basetable group by b -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.cmv_mat_view_4 - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: b (type: varchar(256)) - outputColumnNames: b - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: b (type: varchar(256)) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col0 (type: varchar(256)) - Statistics: Num rows: 5 Data size: 465 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: varchar(256)) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 186 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 186 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select b from cmv_basetable group by b -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_mat_view_4 -#### A masked pattern was here #### -POSTHOOK: query: select b from cmv_basetable group by b -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_mat_view_4 -#### A masked pattern was here #### -alfred -bob -bonnie -calvin -charlie -PREHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_basetable_2 -POSTHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_basetable_2 -PREHOOK: query: insert into cmv_basetable_2 values - (1, 'alfred', 10.30, 2), - (3, 'calvin', 978.76, 3) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@cmv_basetable_2 -POSTHOOK: query: insert into cmv_basetable_2 values - (1, 'alfred', 10.30, 2), - (3, 'calvin', 978.76, 3) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@cmv_basetable_2 -POSTHOOK: Lineage: cmv_basetable_2.a SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_2.b SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_2.c SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_2.d SCRIPT [] -PREHOOK: query: analyze table cmv_basetable_2 compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Output: default@cmv_basetable_2 -#### A masked pattern was here #### -POSTHOOK: query: analyze table cmv_basetable_2 compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Output: default@cmv_basetable_2 -#### A masked pattern was here #### -PREHOOK: query: create materialized view cmv_mat_view_5 enable rewrite -as select cmv_basetable.a, cmv_basetable_2.c - from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) - where cmv_basetable_2.c > 10.0 - group by cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_mat_view_5 -POSTHOOK: query: create materialized view cmv_mat_view_5 enable rewrite -as select cmv_basetable.a, cmv_basetable_2.c - from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) - where cmv_basetable_2.c > 10.0 - group by cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_mat_view_5 -PREHOOK: query: explain -select cmv_basetable.a -from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) -where cmv_basetable_2.c > 10.10 -group by cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -POSTHOOK: query: explain -select cmv_basetable.a -from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) -where cmv_basetable_2.c > 10.10 -group by cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.cmv_mat_view_5 - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (c > 10.1) (type: boolean) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select cmv_basetable.a -from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) -where cmv_basetable_2.c > 10.10 -group by cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Input: default@cmv_mat_view_5 -#### A masked pattern was here #### -POSTHOOK: query: select cmv_basetable.a -from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) -where cmv_basetable_2.c > 10.10 -group by cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Input: default@cmv_mat_view_5 -#### A masked pattern was here #### -1 -3 -PREHOOK: query: explain -select cmv_basetable.a -from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) -where cmv_basetable_2.c > 10.10 -group by cmv_basetable.a -PREHOOK: type: QUERY -POSTHOOK: query: explain -select cmv_basetable.a -from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) -where cmv_basetable_2.c > 10.10 -group by cmv_basetable.a -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.cmv_mat_view_5 - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (c > 10.1) (type: boolean) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: a - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: a (type: int) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select cmv_basetable.a -from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) -where cmv_basetable_2.c > 10.10 -group by cmv_basetable.a -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Input: default@cmv_mat_view_5 -#### A masked pattern was here #### -POSTHOOK: query: select cmv_basetable.a -from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) -where cmv_basetable_2.c > 10.10 -group by cmv_basetable.a -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Input: default@cmv_mat_view_5 -#### A masked pattern was here #### -1 -3 -PREHOOK: query: drop materialized view cmv_mat_view -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_mat_view -PREHOOK: Output: default@cmv_mat_view -POSTHOOK: query: drop materialized view cmv_mat_view -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_mat_view -POSTHOOK: Output: default@cmv_mat_view -PREHOOK: query: drop materialized view cmv_mat_view_2 -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_mat_view_2 -PREHOOK: Output: default@cmv_mat_view_2 -POSTHOOK: query: drop materialized view cmv_mat_view_2 -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_mat_view_2 -POSTHOOK: Output: default@cmv_mat_view_2 -PREHOOK: query: drop materialized view cmv_mat_view_3 -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_mat_view_3 -PREHOOK: Output: default@cmv_mat_view_3 -POSTHOOK: query: drop materialized view cmv_mat_view_3 -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_mat_view_3 -POSTHOOK: Output: default@cmv_mat_view_3 -PREHOOK: query: drop materialized view cmv_mat_view_4 -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_mat_view_4 -PREHOOK: Output: default@cmv_mat_view_4 -POSTHOOK: query: drop materialized view cmv_mat_view_4 -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_mat_view_4 -POSTHOOK: Output: default@cmv_mat_view_4 -PREHOOK: query: drop materialized view cmv_mat_view_5 -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_mat_view_5 -PREHOOK: Output: default@cmv_mat_view_5 -POSTHOOK: query: drop materialized view cmv_mat_view_5 -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_mat_view_5 -POSTHOOK: Output: default@cmv_mat_view_5 diff --git a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_3.q.out b/ql/src/test/results/clientpositive/materialized_view_create_rewrite_3.q.out deleted file mode 100644 index 65614566c9..0000000000 --- a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_3.q.out +++ /dev/null @@ -1,1100 +0,0 @@ -PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_basetable -POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_basetable -PREHOOK: query: insert into cmv_basetable values - (1, 'alfred', 10.30, 2), - (2, 'bob', 3.14, 3), - (2, 'bonnie', 172342.2, 3), - (3, 'calvin', 978.76, 3), - (3, 'charlie', 9.8, 1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@cmv_basetable -POSTHOOK: query: insert into cmv_basetable values - (1, 'alfred', 10.30, 2), - (2, 'bob', 3.14, 3), - (2, 'bonnie', 172342.2, 3), - (3, 'calvin', 978.76, 3), - (3, 'charlie', 9.8, 1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@cmv_basetable -POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] -POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] -POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] -POSTHOOK: Lineage: cmv_basetable.d SCRIPT [] -PREHOOK: query: analyze table cmv_basetable compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Output: default@cmv_basetable -#### A masked pattern was here #### -POSTHOOK: query: analyze table cmv_basetable compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Output: default@cmv_basetable -#### A masked pattern was here #### -PREHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_basetable_2 -POSTHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_basetable_2 -PREHOOK: query: insert into cmv_basetable_2 values - (1, 'alfred', 10.30, 2), - (3, 'calvin', 978.76, 3) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@cmv_basetable_2 -POSTHOOK: query: insert into cmv_basetable_2 values - (1, 'alfred', 10.30, 2), - (3, 'calvin', 978.76, 3) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@cmv_basetable_2 -POSTHOOK: Lineage: cmv_basetable_2.a SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_2.b SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_2.c SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_2.d SCRIPT [] -PREHOOK: query: analyze table cmv_basetable_2 compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Output: default@cmv_basetable_2 -#### A masked pattern was here #### -POSTHOOK: query: analyze table cmv_basetable_2 compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Output: default@cmv_basetable_2 -#### A masked pattern was here #### -PREHOOK: query: EXPLAIN -CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS - SELECT cmv_basetable.a, cmv_basetable_2.c - FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) - WHERE cmv_basetable_2.c > 10.0 - GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: query: EXPLAIN -CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS - SELECT cmv_basetable.a, cmv_basetable_2.c - FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) - WHERE cmv_basetable_2.c > 10.0 - GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-5 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-5 - Stage-6 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: a is not null (type: boolean) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - TableScan - alias: cmv_basetable_2 - Statistics: Num rows: 2 Data size: 484 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((c > 10) and a is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), c (type: decimal(10,2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(10,2)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col2 (type: decimal(10,2)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.cmv_mat_view - - Stage: Stage-0 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-5 - Create View Operator: - Create View - columns: a int, c decimal(10,2) - expanded text: SELECT `cmv_basetable`.`a`, `cmv_basetable_2`.`c` - FROM `default`.`cmv_basetable` JOIN `default`.`cmv_basetable_2` ON (`cmv_basetable`.`a` = `cmv_basetable_2`.`a`) - WHERE `cmv_basetable_2`.`c` > 10.0 - GROUP BY `cmv_basetable`.`a`, `cmv_basetable_2`.`c` - name: default.cmv_mat_view - original text: SELECT cmv_basetable.a, cmv_basetable_2.c - FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) - WHERE cmv_basetable_2.c > 10.0 - GROUP BY cmv_basetable.a, cmv_basetable_2.c - rewrite enabled: true - - Stage: Stage-3 - Stats Work - Basic Stats Work: - - Stage: Stage-6 - Materialized View Work - -PREHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS - SELECT cmv_basetable.a, cmv_basetable_2.c - FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) - WHERE cmv_basetable_2.c > 10.0 - GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_mat_view -POSTHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS - SELECT cmv_basetable.a, cmv_basetable_2.c - FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) - WHERE cmv_basetable_2.c > 10.0 - GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_mat_view -PREHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.cmv_mat_view - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (c > 10.1) (type: boolean) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Input: default@cmv_mat_view -#### A masked pattern was here #### -POSTHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Input: default@cmv_mat_view -#### A masked pattern was here #### -1 -3 -PREHOOK: query: insert into cmv_basetable_2 values - (3, 'charlie', 15.8, 1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@cmv_basetable_2 -POSTHOOK: query: insert into cmv_basetable_2 values - (3, 'charlie', 15.8, 1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@cmv_basetable_2 -POSTHOOK: Lineage: cmv_basetable_2.a SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_2.b SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_2.c SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_2.d SCRIPT [] -PREHOOK: query: analyze table cmv_basetable_2 compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Output: default@cmv_basetable_2 -#### A masked pattern was here #### -POSTHOOK: query: analyze table cmv_basetable_2 compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Output: default@cmv_basetable_2 -#### A masked pattern was here #### -PREHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: a is not null (type: boolean) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - TableScan - alias: cmv_basetable_2 - Statistics: Num rows: 3 Data size: 727 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((c > 10.1) and a is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), c (type: decimal(10,2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(10,2)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col2 (type: decimal(10,2)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -#### A masked pattern was here #### -1 -3 -3 -PREHOOK: query: EXPLAIN -ALTER MATERIALIZED VIEW cmv_mat_view REBUILD -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -ALTER MATERIALIZED VIEW cmv_mat_view REBUILD -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0, Stage-4 - Stage-6 depends on stages: Stage-3 - Stage-4 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: a is not null (type: boolean) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - TableScan - alias: cmv_basetable_2 - Statistics: Num rows: 3 Data size: 727 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((c > 10) and a is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), c (type: decimal(10,2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(10,2)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col2 (type: decimal(10,2)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.cmv_mat_view - Select Operator - expressions: _col0 (type: int), _col1 (type: decimal(10,2)) - outputColumnNames: a, c - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll') - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.cmv_mat_view - - Stage: Stage-3 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: a, c - Column Types: int, decimal(10,2) - Table: default.cmv_mat_view - - Stage: Stage-6 - Materialized View Work - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Output: default@cmv_mat_view -POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Output: default@cmv_mat_view -POSTHOOK: Lineage: cmv_mat_view.a SIMPLE [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), ] -POSTHOOK: Lineage: cmv_mat_view.c SIMPLE [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), ] -PREHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.cmv_mat_view - Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (c > 10.1) (type: boolean) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Input: default@cmv_mat_view -#### A masked pattern was here #### -POSTHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Input: default@cmv_mat_view -#### A masked pattern was here #### -1 -3 -3 -PREHOOK: query: DELETE FROM cmv_basetable_2 WHERE a = 3 -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Output: default@cmv_basetable_2 -POSTHOOK: query: DELETE FROM cmv_basetable_2 WHERE a = 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Output: default@cmv_basetable_2 -PREHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: a is not null (type: boolean) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - TableScan - alias: cmv_basetable_2 - Statistics: Num rows: 42 Data size: 23560 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((c > 10.1) and a is not null) (type: boolean) - Statistics: Num rows: 14 Data size: 7853 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), c (type: decimal(10,2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 7853 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 14 Data size: 7853 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(10,2)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 15 Data size: 8638 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col2 (type: decimal(10,2)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 8638 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) - Statistics: Num rows: 15 Data size: 8638 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 4031 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 4031 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 7 Data size: 4031 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -#### A masked pattern was here #### -1 -PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Output: default@cmv_mat_view -POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Output: default@cmv_mat_view -POSTHOOK: Lineage: cmv_mat_view.a SIMPLE [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), ] -POSTHOOK: Lineage: cmv_mat_view.c SIMPLE [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), ] -PREHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.cmv_mat_view - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (c > 10.1) (type: boolean) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Input: default@cmv_mat_view -#### A masked pattern was here #### -POSTHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Input: default@cmv_mat_view -#### A masked pattern was here #### -1 -PREHOOK: query: create table cmv_irrelevant_table (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_irrelevant_table -POSTHOOK: query: create table cmv_irrelevant_table (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_irrelevant_table -PREHOOK: query: insert into cmv_irrelevant_table values - (1, 'alfred', 10.30, 2), - (3, 'charlie', 9.8, 1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@cmv_irrelevant_table -POSTHOOK: query: insert into cmv_irrelevant_table values - (1, 'alfred', 10.30, 2), - (3, 'charlie', 9.8, 1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@cmv_irrelevant_table -POSTHOOK: Lineage: cmv_irrelevant_table.a SCRIPT [] -POSTHOOK: Lineage: cmv_irrelevant_table.b SCRIPT [] -POSTHOOK: Lineage: cmv_irrelevant_table.c SCRIPT [] -POSTHOOK: Lineage: cmv_irrelevant_table.d SCRIPT [] -PREHOOK: query: analyze table cmv_irrelevant_table compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_irrelevant_table -PREHOOK: Output: default@cmv_irrelevant_table -#### A masked pattern was here #### -POSTHOOK: query: analyze table cmv_irrelevant_table compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_irrelevant_table -POSTHOOK: Output: default@cmv_irrelevant_table -#### A masked pattern was here #### -PREHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.cmv_mat_view - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (c > 10.1) (type: boolean) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Input: default@cmv_mat_view -#### A masked pattern was here #### -POSTHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Input: default@cmv_mat_view -#### A masked pattern was here #### -1 -PREHOOK: query: drop materialized view cmv_mat_view -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_mat_view -PREHOOK: Output: default@cmv_mat_view -POSTHOOK: query: drop materialized view cmv_mat_view -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_mat_view -POSTHOOK: Output: default@cmv_mat_view -PREHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: a is not null (type: boolean) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - TableScan - alias: cmv_basetable_2 - Statistics: Num rows: 42 Data size: 23560 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((c > 10.1) and a is not null) (type: boolean) - Statistics: Num rows: 14 Data size: 7853 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), c (type: decimal(10,2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 14 Data size: 7853 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 14 Data size: 7853 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(10,2)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 15 Data size: 8638 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col2 (type: decimal(10,2)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 15 Data size: 8638 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) - Statistics: Num rows: 15 Data size: 8638 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 7 Data size: 4031 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 7 Data size: 4031 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 7 Data size: 4031 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -#### A masked pattern was here #### -1 diff --git a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_4.q.out b/ql/src/test/results/clientpositive/materialized_view_create_rewrite_4.q.out deleted file mode 100644 index 48c0ecb23f..0000000000 --- a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_4.q.out +++ /dev/null @@ -1,886 +0,0 @@ -PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_basetable -POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_basetable -PREHOOK: query: insert into cmv_basetable values - (1, 'alfred', 10.30, 2), - (2, 'bob', 3.14, 3), - (2, 'bonnie', 172342.2, 3), - (3, 'calvin', 978.76, 3), - (3, 'charlie', 9.8, 1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@cmv_basetable -POSTHOOK: query: insert into cmv_basetable values - (1, 'alfred', 10.30, 2), - (2, 'bob', 3.14, 3), - (2, 'bonnie', 172342.2, 3), - (3, 'calvin', 978.76, 3), - (3, 'charlie', 9.8, 1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@cmv_basetable -POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] -POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] -POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] -POSTHOOK: Lineage: cmv_basetable.d SCRIPT [] -PREHOOK: query: analyze table cmv_basetable compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Output: default@cmv_basetable -#### A masked pattern was here #### -POSTHOOK: query: analyze table cmv_basetable compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Output: default@cmv_basetable -#### A masked pattern was here #### -PREHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_basetable_2 -POSTHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_basetable_2 -PREHOOK: query: insert into cmv_basetable_2 values - (1, 'alfred', 10.30, 2), - (3, 'calvin', 978.76, 3) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@cmv_basetable_2 -POSTHOOK: query: insert into cmv_basetable_2 values - (1, 'alfred', 10.30, 2), - (3, 'calvin', 978.76, 3) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@cmv_basetable_2 -POSTHOOK: Lineage: cmv_basetable_2.a SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_2.b SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_2.c SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_2.d SCRIPT [] -PREHOOK: query: analyze table cmv_basetable_2 compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Output: default@cmv_basetable_2 -#### A masked pattern was here #### -POSTHOOK: query: analyze table cmv_basetable_2 compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Output: default@cmv_basetable_2 -#### A masked pattern was here #### -PREHOOK: query: EXPLAIN -CREATE MATERIALIZED VIEW cmv_mat_view AS - SELECT cmv_basetable.a, cmv_basetable_2.c - FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) - WHERE cmv_basetable_2.c > 10.0 - GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: query: EXPLAIN -CREATE MATERIALIZED VIEW cmv_mat_view AS - SELECT cmv_basetable.a, cmv_basetable_2.c - FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) - WHERE cmv_basetable_2.c > 10.0 - GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-5 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-5 - Stage-6 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: a is not null (type: boolean) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - TableScan - alias: cmv_basetable_2 - Statistics: Num rows: 2 Data size: 484 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((c > 10) and a is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), c (type: decimal(10,2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(10,2)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col2 (type: decimal(10,2)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.cmv_mat_view - - Stage: Stage-0 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - - Stage: Stage-5 - Create View Operator: - Create View - columns: a int, c decimal(10,2) - expanded text: SELECT `cmv_basetable`.`a`, `cmv_basetable_2`.`c` - FROM `default`.`cmv_basetable` JOIN `default`.`cmv_basetable_2` ON (`cmv_basetable`.`a` = `cmv_basetable_2`.`a`) - WHERE `cmv_basetable_2`.`c` > 10.0 - GROUP BY `cmv_basetable`.`a`, `cmv_basetable_2`.`c` - name: default.cmv_mat_view - original text: SELECT cmv_basetable.a, cmv_basetable_2.c - FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) - WHERE cmv_basetable_2.c > 10.0 - GROUP BY cmv_basetable.a, cmv_basetable_2.c - - Stage: Stage-3 - Stats Work - Basic Stats Work: - - Stage: Stage-6 - Materialized View Work - -PREHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view AS - SELECT cmv_basetable.a, cmv_basetable_2.c - FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) - WHERE cmv_basetable_2.c > 10.0 - GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Output: database:default -PREHOOK: Output: default@cmv_mat_view -POSTHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view AS - SELECT cmv_basetable.a, cmv_basetable_2.c - FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) - WHERE cmv_basetable_2.c > 10.0 - GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@cmv_mat_view -PREHOOK: query: DESCRIBE FORMATTED cmv_mat_view -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@cmv_mat_view -POSTHOOK: query: DESCRIBE FORMATTED cmv_mat_view -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@cmv_mat_view -# col_name data_type comment -a int -c decimal(10,2) - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MATERIALIZED_VIEW -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - numFiles 1 - numRows 2 - rawDataSize 232 - totalSize 325 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde -InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] - -# View Information -View Original Text: SELECT cmv_basetable.a, cmv_basetable_2.c - FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) - WHERE cmv_basetable_2.c > 10.0 - GROUP BY cmv_basetable.a, cmv_basetable_2.c -View Expanded Text: SELECT `cmv_basetable`.`a`, `cmv_basetable_2`.`c` - FROM `default`.`cmv_basetable` JOIN `default`.`cmv_basetable_2` ON (`cmv_basetable`.`a` = `cmv_basetable_2`.`a`) - WHERE `cmv_basetable_2`.`c` > 10.0 - GROUP BY `cmv_basetable`.`a`, `cmv_basetable_2`.`c` -View Rewrite Enabled: No -PREHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: a is not null (type: boolean) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - TableScan - alias: cmv_basetable_2 - Statistics: Num rows: 2 Data size: 484 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((c > 10.1) and a is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), c (type: decimal(10,2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(10,2)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col2 (type: decimal(10,2)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -#### A masked pattern was here #### -1 -3 -PREHOOK: query: insert into cmv_basetable_2 values - (3, 'charlie', 15.8, 1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@cmv_basetable_2 -POSTHOOK: query: insert into cmv_basetable_2 values - (3, 'charlie', 15.8, 1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@cmv_basetable_2 -POSTHOOK: Lineage: cmv_basetable_2.a SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_2.b SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_2.c SCRIPT [] -POSTHOOK: Lineage: cmv_basetable_2.d SCRIPT [] -PREHOOK: query: analyze table cmv_basetable_2 compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Output: default@cmv_basetable_2 -#### A masked pattern was here #### -POSTHOOK: query: analyze table cmv_basetable_2 compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Output: default@cmv_basetable_2 -#### A masked pattern was here #### -PREHOOK: query: EXPLAIN -ALTER MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE -PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE -POSTHOOK: query: EXPLAIN -ALTER MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE -POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Alter Materialized View Operator: - Alter Materialized View - name: default.cmv_mat_view - operation: UPDATE_REWRITE_FLAG - -PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE -PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE -PREHOOK: Input: default@cmv_mat_view -PREHOOK: Output: default@cmv_mat_view -POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE -POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE -POSTHOOK: Input: default@cmv_mat_view -POSTHOOK: Output: default@cmv_mat_view -PREHOOK: query: DESCRIBE FORMATTED cmv_mat_view -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@cmv_mat_view -POSTHOOK: query: DESCRIBE FORMATTED cmv_mat_view -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@cmv_mat_view -# col_name data_type comment -a int -c decimal(10,2) - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MATERIALIZED_VIEW -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - numFiles 1 - numRows 2 - rawDataSize 232 - totalSize 325 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde -InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] - -# View Information -View Original Text: SELECT cmv_basetable.a, cmv_basetable_2.c - FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) - WHERE cmv_basetable_2.c > 10.0 - GROUP BY cmv_basetable.a, cmv_basetable_2.c -View Expanded Text: SELECT `cmv_basetable`.`a`, `cmv_basetable_2`.`c` - FROM `default`.`cmv_basetable` JOIN `default`.`cmv_basetable_2` ON (`cmv_basetable`.`a` = `cmv_basetable_2`.`a`) - WHERE `cmv_basetable_2`.`c` > 10.0 - GROUP BY `cmv_basetable`.`a`, `cmv_basetable_2`.`c` -View Rewrite Enabled: Yes -PREHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: a is not null (type: boolean) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - TableScan - alias: cmv_basetable_2 - Statistics: Num rows: 3 Data size: 727 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((c > 10.1) and a is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), c (type: decimal(10,2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(10,2)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col2 (type: decimal(10,2)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -#### A masked pattern was here #### -POSTHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -#### A masked pattern was here #### -1 -3 -3 -PREHOOK: query: EXPLAIN -ALTER MATERIALIZED VIEW cmv_mat_view REBUILD -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -ALTER MATERIALIZED VIEW cmv_mat_view REBUILD -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0, Stage-4 - Stage-6 depends on stages: Stage-3 - Stage-4 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: cmv_basetable - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: a is not null (type: boolean) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE - TableScan - alias: cmv_basetable_2 - Statistics: Num rows: 3 Data size: 727 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((c > 10) and a is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int), c (type: decimal(10,2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 242 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(10,2)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col2 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: int), _col2 (type: decimal(10,2)) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) - Statistics: Num rows: 5 Data size: 1325 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.cmv_mat_view - Select Operator - expressions: _col0 (type: int), _col1 (type: decimal(10,2)) - outputColumnNames: a, c - Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll') - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Move Operator - tables: - replace: true - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.cmv_mat_view - - Stage: Stage-3 - Stats Work - Basic Stats Work: - Column Stats Desc: - Columns: a, c - Column Types: int, decimal(10,2) - Table: default.cmv_mat_view - - Stage: Stage-6 - Materialized View Work - - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 1056 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: struct), _col1 (type: struct) - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 1088 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - -PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Output: default@cmv_mat_view -POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Output: default@cmv_mat_view -POSTHOOK: Lineage: cmv_mat_view.a SIMPLE [(cmv_basetable)cmv_basetable.FieldSchema(name:a, type:int, comment:null), ] -POSTHOOK: Lineage: cmv_mat_view.c SIMPLE [(cmv_basetable_2)cmv_basetable_2.FieldSchema(name:c, type:decimal(10,2), comment:null), ] -PREHOOK: query: DESCRIBE FORMATTED cmv_mat_view -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@cmv_mat_view -POSTHOOK: query: DESCRIBE FORMATTED cmv_mat_view -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@cmv_mat_view -# col_name data_type comment -a int -c decimal(10,2) - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MATERIALIZED_VIEW -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"c\":\"true\"}} - numFiles 1 - numRows 3 - rawDataSize 348 - totalSize 332 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde -InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] - -# View Information -View Original Text: SELECT cmv_basetable.a, cmv_basetable_2.c - FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) - WHERE cmv_basetable_2.c > 10.0 - GROUP BY cmv_basetable.a, cmv_basetable_2.c -View Expanded Text: SELECT `cmv_basetable`.`a`, `cmv_basetable_2`.`c` - FROM `default`.`cmv_basetable` JOIN `default`.`cmv_basetable_2` ON (`cmv_basetable`.`a` = `cmv_basetable_2`.`a`) - WHERE `cmv_basetable_2`.`c` > 10.0 - GROUP BY `cmv_basetable`.`a`, `cmv_basetable_2`.`c` -View Rewrite Enabled: Yes -PREHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN -SELECT cmv_basetable.a -FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.cmv_mat_view - Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (c > 10.1) (type: boolean) - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: a (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -PREHOOK: type: QUERY -PREHOOK: Input: default@cmv_basetable -PREHOOK: Input: default@cmv_basetable_2 -PREHOOK: Input: default@cmv_mat_view -#### A masked pattern was here #### -POSTHOOK: query: SELECT cmv_basetable.a -FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) -WHERE cmv_basetable_2.c > 10.10 -GROUP BY cmv_basetable.a, cmv_basetable_2.c -POSTHOOK: type: QUERY -POSTHOOK: Input: default@cmv_basetable -POSTHOOK: Input: default@cmv_basetable_2 -POSTHOOK: Input: default@cmv_mat_view -#### A masked pattern was here #### -1 -3 -3 -PREHOOK: query: drop materialized view cmv_mat_view -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@cmv_mat_view -PREHOOK: Output: default@cmv_mat_view -POSTHOOK: query: drop materialized view cmv_mat_view -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@cmv_mat_view -POSTHOOK: Output: default@cmv_mat_view diff --git a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_multi_db.q.out b/ql/src/test/results/clientpositive/materialized_view_create_rewrite_multi_db.q.out deleted file mode 100644 index 32b408f5e4..0000000000 --- a/ql/src/test/results/clientpositive/materialized_view_create_rewrite_multi_db.q.out +++ /dev/null @@ -1,175 +0,0 @@ -PREHOOK: query: create database db1 -PREHOOK: type: CREATEDATABASE -PREHOOK: Output: database:db1 -POSTHOOK: query: create database db1 -POSTHOOK: type: CREATEDATABASE -POSTHOOK: Output: database:db1 -PREHOOK: query: use db1 -PREHOOK: type: SWITCHDATABASE -PREHOOK: Input: database:db1 -POSTHOOK: query: use db1 -POSTHOOK: type: SWITCHDATABASE -POSTHOOK: Input: database:db1 -PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:db1 -PREHOOK: Output: db1@cmv_basetable -POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:db1 -POSTHOOK: Output: db1@cmv_basetable -PREHOOK: query: insert into cmv_basetable values - (1, 'alfred', 10.30, 2), - (2, 'bob', 3.14, 3), - (2, 'bonnie', 172342.2, 3), - (3, 'calvin', 978.76, 3), - (3, 'charlie', 9.8, 1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: db1@cmv_basetable -POSTHOOK: query: insert into cmv_basetable values - (1, 'alfred', 10.30, 2), - (2, 'bob', 3.14, 3), - (2, 'bonnie', 172342.2, 3), - (3, 'calvin', 978.76, 3), - (3, 'charlie', 9.8, 1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: db1@cmv_basetable -POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] -POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] -POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] -POSTHOOK: Lineage: cmv_basetable.d SCRIPT [] -PREHOOK: query: analyze table cmv_basetable compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: db1@cmv_basetable -PREHOOK: Output: db1@cmv_basetable -#### A masked pattern was here #### -POSTHOOK: query: analyze table cmv_basetable compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: db1@cmv_basetable -POSTHOOK: Output: db1@cmv_basetable -#### A masked pattern was here #### -PREHOOK: query: create database db2 -PREHOOK: type: CREATEDATABASE -PREHOOK: Output: database:db2 -POSTHOOK: query: create database db2 -POSTHOOK: type: CREATEDATABASE -POSTHOOK: Output: database:db2 -PREHOOK: query: use db2 -PREHOOK: type: SWITCHDATABASE -PREHOOK: Input: database:db2 -POSTHOOK: query: use db2 -POSTHOOK: type: SWITCHDATABASE -POSTHOOK: Input: database:db2 -PREHOOK: query: create materialized view cmv_mat_view enable rewrite -as select a, b, c from db1.cmv_basetable where a = 2 -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: db1@cmv_basetable -PREHOOK: Output: database:db2 -PREHOOK: Output: db2@cmv_mat_view -POSTHOOK: query: create materialized view cmv_mat_view enable rewrite -as select a, b, c from db1.cmv_basetable where a = 2 -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: db1@cmv_basetable -POSTHOOK: Output: database:db2 -POSTHOOK: Output: db2@cmv_mat_view -PREHOOK: query: select * from cmv_mat_view -PREHOOK: type: QUERY -PREHOOK: Input: db2@cmv_mat_view -#### A masked pattern was here #### -POSTHOOK: query: select * from cmv_mat_view -POSTHOOK: type: QUERY -POSTHOOK: Input: db2@cmv_mat_view -#### A masked pattern was here #### -2 bob 3.14 -2 bonnie 172342.20 -PREHOOK: query: show tblproperties cmv_mat_view -PREHOOK: type: SHOW_TBLPROPERTIES -POSTHOOK: query: show tblproperties cmv_mat_view -POSTHOOK: type: SHOW_TBLPROPERTIES -COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} -numFiles 1 -numRows 2 -rawDataSize 408 -totalSize 453 -#### A masked pattern was here #### -PREHOOK: query: create materialized view if not exists cmv_mat_view2 enable rewrite -as select a, c from db1.cmv_basetable where a = 3 -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: db1@cmv_basetable -PREHOOK: Output: database:db2 -PREHOOK: Output: db2@cmv_mat_view2 -POSTHOOK: query: create materialized view if not exists cmv_mat_view2 enable rewrite -as select a, c from db1.cmv_basetable where a = 3 -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: db1@cmv_basetable -POSTHOOK: Output: database:db2 -POSTHOOK: Output: db2@cmv_mat_view2 -PREHOOK: query: select * from cmv_mat_view2 -PREHOOK: type: QUERY -PREHOOK: Input: db2@cmv_mat_view2 -#### A masked pattern was here #### -POSTHOOK: query: select * from cmv_mat_view2 -POSTHOOK: type: QUERY -POSTHOOK: Input: db2@cmv_mat_view2 -#### A masked pattern was here #### -3 978.76 -3 9.80 -PREHOOK: query: show tblproperties cmv_mat_view2 -PREHOOK: type: SHOW_TBLPROPERTIES -POSTHOOK: query: show tblproperties cmv_mat_view2 -POSTHOOK: type: SHOW_TBLPROPERTIES -COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} -numFiles 1 -numRows 2 -rawDataSize 232 -totalSize 322 -#### A masked pattern was here #### -PREHOOK: query: create database db3 -PREHOOK: type: CREATEDATABASE -PREHOOK: Output: database:db3 -POSTHOOK: query: create database db3 -POSTHOOK: type: CREATEDATABASE -POSTHOOK: Output: database:db3 -PREHOOK: query: use db3 -PREHOOK: type: SWITCHDATABASE -PREHOOK: Input: database:db3 -POSTHOOK: query: use db3 -POSTHOOK: type: SWITCHDATABASE -POSTHOOK: Input: database:db3 -PREHOOK: query: explain -select a, c from db1.cmv_basetable where a = 3 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select a, c from db1.cmv_basetable where a = 3 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - TableScan - alias: db2.cmv_mat_view2 - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: 3 (type: int), c (type: decimal(10,2)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE - ListSink - -PREHOOK: query: select a, c from db1.cmv_basetable where a = 3 -PREHOOK: type: QUERY -PREHOOK: Input: db1@cmv_basetable -PREHOOK: Input: db2@cmv_mat_view2 -#### A masked pattern was here #### -POSTHOOK: query: select a, c from db1.cmv_basetable where a = 3 -POSTHOOK: type: QUERY -POSTHOOK: Input: db1@cmv_basetable -POSTHOOK: Input: db2@cmv_mat_view2 -#### A masked pattern was here #### -3 978.76 -3 9.80 diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_7.q.out b/ql/src/test/results/clientpositive/materialized_view_rewrite_7.q.out deleted file mode 100644 index 486a50d55a..0000000000 --- a/ql/src/test/results/clientpositive/materialized_view_rewrite_7.q.out +++ /dev/null @@ -1,1036 +0,0 @@ -PREHOOK: query: create table emps ( - empid int, - deptno int, - name varchar(256), - salary float, - commission int) -stored as orc TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@emps -POSTHOOK: query: create table emps ( - empid int, - deptno int, - name varchar(256), - salary float, - commission int) -stored as orc TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@emps -PREHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), - (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@emps -POSTHOOK: query: insert into emps values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), - (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@emps -POSTHOOK: Lineage: emps.commission SCRIPT [] -POSTHOOK: Lineage: emps.deptno SCRIPT [] -POSTHOOK: Lineage: emps.empid SCRIPT [] -POSTHOOK: Lineage: emps.name SCRIPT [] -POSTHOOK: Lineage: emps.salary SCRIPT [] -PREHOOK: query: analyze table emps compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@emps -PREHOOK: Output: default@emps -#### A masked pattern was here #### -POSTHOOK: query: analyze table emps compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@emps -POSTHOOK: Output: default@emps -#### A masked pattern was here #### -PREHOOK: query: create table depts ( - deptno int, - name varchar(256), - locationid int) -stored as orc TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@depts -POSTHOOK: query: create table depts ( - deptno int, - name varchar(256), - locationid int) -stored as orc TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@depts -PREHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@depts -POSTHOOK: query: insert into depts values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@depts -POSTHOOK: Lineage: depts.deptno SCRIPT [] -POSTHOOK: Lineage: depts.locationid SCRIPT [] -POSTHOOK: Lineage: depts.name SCRIPT [] -PREHOOK: query: analyze table depts compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@depts -PREHOOK: Output: default@depts -#### A masked pattern was here #### -POSTHOOK: query: analyze table depts compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@depts -POSTHOOK: Output: default@depts -#### A masked pattern was here #### -PREHOOK: query: create table dependents ( - empid int, - name varchar(256)) -stored as orc TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dependents -POSTHOOK: query: create table dependents ( - empid int, - name varchar(256)) -stored as orc TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dependents -PREHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@dependents -POSTHOOK: query: insert into dependents values (10, 'Michael'), (10, 'Jane') -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@dependents -POSTHOOK: Lineage: dependents.empid SCRIPT [] -POSTHOOK: Lineage: dependents.name SCRIPT [] -PREHOOK: query: analyze table dependents compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@dependents -PREHOOK: Output: default@dependents -#### A masked pattern was here #### -POSTHOOK: query: analyze table dependents compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dependents -POSTHOOK: Output: default@dependents -#### A masked pattern was here #### -PREHOOK: query: create table locations ( - locationid int, - name varchar(256)) -stored as orc TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@locations -POSTHOOK: query: create table locations ( - locationid int, - name varchar(256)) -stored as orc TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@locations -PREHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@locations -POSTHOOK: query: insert into locations values (10, 'San Francisco'), (10, 'San Diego') -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@locations -POSTHOOK: Lineage: locations.locationid SCRIPT [] -POSTHOOK: Lineage: locations.name SCRIPT [] -PREHOOK: query: analyze table locations compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@locations -PREHOOK: Output: default@locations -#### A masked pattern was here #### -POSTHOOK: query: analyze table locations compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@locations -POSTHOOK: Output: default@locations -#### A masked pattern was here #### -PREHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely -PREHOOK: type: ALTERTABLE_ADDCONSTRAINT -POSTHOOK: query: alter table emps add constraint pk1 primary key (empid) disable novalidate rely -POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT -PREHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely -PREHOOK: type: ALTERTABLE_ADDCONSTRAINT -POSTHOOK: query: alter table depts add constraint pk2 primary key (deptno) disable novalidate rely -POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT -PREHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely -PREHOOK: type: ALTERTABLE_ADDCONSTRAINT -POSTHOOK: query: alter table dependents add constraint pk3 primary key (empid) disable novalidate rely -POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT -PREHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely -PREHOOK: type: ALTERTABLE_ADDCONSTRAINT -POSTHOOK: query: alter table locations add constraint pk4 primary key (locationid) disable novalidate rely -POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT -PREHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely -PREHOOK: type: ALTERTABLE_ADDCONSTRAINT -POSTHOOK: query: alter table emps add constraint fk1 foreign key (deptno) references depts(deptno) disable novalidate rely -POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT -PREHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely -PREHOOK: type: ALTERTABLE_ADDCONSTRAINT -POSTHOOK: query: alter table depts add constraint fk2 foreign key (locationid) references locations(locationid) disable novalidate rely -POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT -PREHOOK: query: alter table emps change column deptno deptno int constraint nn1 not null disable novalidate rely -PREHOOK: type: ALTERTABLE_RENAMECOL -PREHOOK: Input: default@emps -PREHOOK: Output: default@emps -POSTHOOK: query: alter table emps change column deptno deptno int constraint nn1 not null disable novalidate rely -POSTHOOK: type: ALTERTABLE_RENAMECOL -POSTHOOK: Input: default@emps -POSTHOOK: Output: default@emps -PREHOOK: query: alter table depts change column locationid locationid int constraint nn2 not null disable novalidate rely -PREHOOK: type: ALTERTABLE_RENAMECOL -PREHOOK: Input: default@depts -PREHOOK: Output: default@depts -POSTHOOK: query: alter table depts change column locationid locationid int constraint nn2 not null disable novalidate rely -POSTHOOK: type: ALTERTABLE_RENAMECOL -POSTHOOK: Input: default@depts -POSTHOOK: Output: default@depts -PREHOOK: query: create materialized view mv1 enable rewrite as -select depts.deptno, dependents.empid -from depts -join dependents on (depts.name = dependents.name) -join locations on (locations.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 11 -group by depts.deptno, dependents.empid -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@dependents -PREHOOK: Input: default@depts -PREHOOK: Input: default@emps -PREHOOK: Input: default@locations -PREHOOK: Output: database:default -PREHOOK: Output: default@mv1 -POSTHOOK: query: create materialized view mv1 enable rewrite as -select depts.deptno, dependents.empid -from depts -join dependents on (depts.name = dependents.name) -join locations on (locations.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 11 -group by depts.deptno, dependents.empid -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@dependents -POSTHOOK: Input: default@depts -POSTHOOK: Input: default@emps -POSTHOOK: Input: default@locations -POSTHOOK: Output: database:default -POSTHOOK: Output: default@mv1 -PREHOOK: query: analyze table mv1 compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@mv1 -PREHOOK: Output: default@mv1 -#### A masked pattern was here #### -POSTHOOK: query: analyze table mv1 compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@mv1 -POSTHOOK: Output: default@mv1 -#### A masked pattern was here #### -PREHOOK: query: explain -select dependents.empid, depts.deptno -from depts -join dependents on (depts.name = dependents.name) -join locations on (locations.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 10 -group by dependents.empid, depts.deptno -PREHOOK: type: QUERY -POSTHOOK: query: explain -select dependents.empid, depts.deptno -from depts -join dependents on (depts.name = dependents.name) -join locations on (locations.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 10 -group by dependents.empid, depts.deptno -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-5 is a root stage - Stage-1 depends on stages: Stage-5 - Stage-2 depends on stages: Stage-1 - Stage-3 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - alias: depts - Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((deptno <= 11) and (deptno > 10) and name is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int), name (type: varchar(256)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: varchar(256)) - TableScan - alias: emps - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((deptno <= 11) and (deptno > 10)) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: dependents - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: name is not null (type: boolean) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int), name (type: varchar(256)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col1 (type: varchar(256)) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - TableScan - alias: locations - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: name is not null (type: boolean) - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: name (type: varchar(256)) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col0 (type: varchar(256)) - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - Reduce Output Operator - key expressions: _col1 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col1 (type: varchar(256)) - Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 _col1 (type: varchar(256)) - 1 _col0 (type: varchar(256)) - 2 _col1 (type: varchar(256)) - outputColumnNames: _col0, _col3 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int), _col3 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: default.mv1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int), deptno (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: int), _col1 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select dependents.empid, depts.deptno -from depts -join dependents on (depts.name = dependents.name) -join locations on (locations.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 10 -group by dependents.empid, depts.deptno -PREHOOK: type: QUERY -PREHOOK: Input: default@dependents -PREHOOK: Input: default@depts -PREHOOK: Input: default@emps -PREHOOK: Input: default@locations -PREHOOK: Input: default@mv1 -#### A masked pattern was here #### -POSTHOOK: query: select dependents.empid, depts.deptno -from depts -join dependents on (depts.name = dependents.name) -join locations on (locations.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 10 -group by dependents.empid, depts.deptno -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dependents -POSTHOOK: Input: default@depts -POSTHOOK: Input: default@emps -POSTHOOK: Input: default@locations -POSTHOOK: Input: default@mv1 -#### A masked pattern was here #### -PREHOOK: query: drop materialized view mv1 -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@mv1 -PREHOOK: Output: default@mv1 -POSTHOOK: query: drop materialized view mv1 -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@mv1 -POSTHOOK: Output: default@mv1 -PREHOOK: query: create materialized view mv1 enable rewrite as -select depts.deptno, dependents.empid, count(emps.salary) as s -from depts -join dependents on (depts.name = dependents.name) -join locations on (locations.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 11 and depts.deptno < 19 -group by depts.deptno, dependents.empid -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@dependents -PREHOOK: Input: default@depts -PREHOOK: Input: default@emps -PREHOOK: Input: default@locations -PREHOOK: Output: database:default -PREHOOK: Output: default@mv1 -POSTHOOK: query: create materialized view mv1 enable rewrite as -select depts.deptno, dependents.empid, count(emps.salary) as s -from depts -join dependents on (depts.name = dependents.name) -join locations on (locations.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 11 and depts.deptno < 19 -group by depts.deptno, dependents.empid -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@dependents -POSTHOOK: Input: default@depts -POSTHOOK: Input: default@emps -POSTHOOK: Input: default@locations -POSTHOOK: Output: database:default -POSTHOOK: Output: default@mv1 -PREHOOK: query: analyze table mv1 compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@mv1 -PREHOOK: Output: default@mv1 -#### A masked pattern was here #### -POSTHOOK: query: analyze table mv1 compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@mv1 -POSTHOOK: Output: default@mv1 -#### A masked pattern was here #### -PREHOOK: query: explain -select dependents.empid, count(emps.salary) + 1 -from depts -join dependents on (depts.name = dependents.name) -join locations on (locations.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 10 and depts.deptno < 20 -group by dependents.empid -PREHOOK: type: QUERY -POSTHOOK: query: explain -select dependents.empid, count(emps.salary) + 1 -from depts -join dependents on (depts.name = dependents.name) -join locations on (locations.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 10 and depts.deptno < 20 -group by dependents.empid -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-5 is a root stage - Stage-1 depends on stages: Stage-5 - Stage-2 depends on stages: Stage-1 - Stage-3 depends on stages: Stage-2, Stage-7 - Stage-7 is a root stage - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - alias: emps - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (((deptno <= 11) or (deptno >= 19)) and (deptno < 20) and (deptno > 10)) (type: boolean) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int), salary (type: float) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: float) - TableScan - alias: depts - Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (((deptno <= 11) or (deptno >= 19)) and (deptno < 20) and (deptno > 10) and name is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int), name (type: varchar(256)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: varchar(256)) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col1, _col3 - Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: dependents - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: name is not null (type: boolean) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int), name (type: varchar(256)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col1 (type: varchar(256)) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - TableScan - alias: locations - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: name is not null (type: boolean) - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: name (type: varchar(256)) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col0 (type: varchar(256)) - Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - Reduce Output Operator - key expressions: _col3 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col3 (type: varchar(256)) - Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: float) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 _col1 (type: varchar(256)) - 1 _col0 (type: varchar(256)) - 2 _col3 (type: varchar(256)) - outputColumnNames: _col0, _col4 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count(_col4) - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: $sum0(_col1) - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - TableScan - Union - Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: $sum0(_col1) - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: $sum0(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: int), (_col1 + 1L) (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Map Reduce - Map Operator Tree: - TableScan - alias: default.mv1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int), s (type: bigint) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: $sum0(_col1) - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: - Group By Operator - aggregations: $sum0(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select dependents.empid, count(emps.salary) + 1 -from depts -join dependents on (depts.name = dependents.name) -join locations on (locations.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 10 and depts.deptno < 20 -group by dependents.empid -PREHOOK: type: QUERY -PREHOOK: Input: default@dependents -PREHOOK: Input: default@depts -PREHOOK: Input: default@emps -PREHOOK: Input: default@locations -PREHOOK: Input: default@mv1 -#### A masked pattern was here #### -POSTHOOK: query: select dependents.empid, count(emps.salary) + 1 -from depts -join dependents on (depts.name = dependents.name) -join locations on (locations.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 10 and depts.deptno < 20 -group by dependents.empid -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dependents -POSTHOOK: Input: default@depts -POSTHOOK: Input: default@emps -POSTHOOK: Input: default@locations -POSTHOOK: Input: default@mv1 -#### A masked pattern was here #### -PREHOOK: query: drop materialized view mv1 -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@mv1 -PREHOOK: Output: default@mv1 -POSTHOOK: query: drop materialized view mv1 -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@mv1 -POSTHOOK: Output: default@mv1 -PREHOOK: query: create materialized view mv1 enable rewrite as -select depts.deptno, dependents.empid -from depts -join dependents on (depts.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno >= 10 -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@dependents -PREHOOK: Input: default@depts -PREHOOK: Input: default@emps -PREHOOK: Output: database:default -PREHOOK: Output: default@mv1 -POSTHOOK: query: create materialized view mv1 enable rewrite as -select depts.deptno, dependents.empid -from depts -join dependents on (depts.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno >= 10 -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@dependents -POSTHOOK: Input: default@depts -POSTHOOK: Input: default@emps -POSTHOOK: Output: database:default -POSTHOOK: Output: default@mv1 -PREHOOK: query: analyze table mv1 compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@mv1 -PREHOOK: Output: default@mv1 -#### A masked pattern was here #### -POSTHOOK: query: analyze table mv1 compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@mv1 -POSTHOOK: Output: default@mv1 -#### A masked pattern was here #### -PREHOOK: query: explain -select dependents.empid -from depts -join dependents on (depts.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 0 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select dependents.empid -from depts -join dependents on (depts.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 0 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-3 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: depts - Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((deptno < 10) and (deptno > 0) and name is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int), name (type: varchar(256)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 93 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: varchar(256)) - TableScan - alias: emps - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: ((deptno < 10) and (deptno > 0)) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: deptno (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col1 - Statistics: Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col1 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col1 (type: varchar(256)) - Statistics: Num rows: 1 Data size: 89 Basic stats: COMPLETE Column stats: COMPLETE - TableScan - alias: dependents - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: name is not null (type: boolean) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int), name (type: varchar(256)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: varchar(256)) - sort order: + - Map-reduce partition columns: _col1 (type: varchar(256)) - Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int) - Reduce Operator Tree: - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: varchar(256)) - 1 _col1 (type: varchar(256)) - outputColumnNames: _col3 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - Union - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TableScan - alias: default.mv1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: empid (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE - Union - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select dependents.empid -from depts -join dependents on (depts.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 0 -PREHOOK: type: QUERY -PREHOOK: Input: default@dependents -PREHOOK: Input: default@depts -PREHOOK: Input: default@emps -PREHOOK: Input: default@mv1 -#### A masked pattern was here #### -POSTHOOK: query: select dependents.empid -from depts -join dependents on (depts.name = dependents.name) -join emps on (emps.deptno = depts.deptno) -where depts.deptno > 0 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dependents -POSTHOOK: Input: default@depts -POSTHOOK: Input: default@emps -POSTHOOK: Input: default@mv1 -#### A masked pattern was here #### -PREHOOK: query: drop materialized view mv1 -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@mv1 -PREHOOK: Output: default@mv1 -POSTHOOK: query: drop materialized view mv1 -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@mv1 -POSTHOOK: Output: default@mv1 diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_ssb.q.out b/ql/src/test/results/clientpositive/materialized_view_rewrite_ssb.q.out deleted file mode 100644 index bc42df0227..0000000000 --- a/ql/src/test/results/clientpositive/materialized_view_rewrite_ssb.q.out +++ /dev/null @@ -1,1975 +0,0 @@ -PREHOOK: query: CREATE TABLE `customer_ext`( - `c_custkey` bigint, - `c_name` string, - `c_address` string, - `c_city` string, - `c_nation` string, - `c_region` string, - `c_phone` string, - `c_mktsegment` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@customer_ext -POSTHOOK: query: CREATE TABLE `customer_ext`( - `c_custkey` bigint, - `c_name` string, - `c_address` string, - `c_city` string, - `c_nation` string, - `c_region` string, - `c_phone` string, - `c_mktsegment` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@customer_ext -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/customer/' into table `customer_ext` -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@customer_ext -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/customer/' into table `customer_ext` -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@customer_ext -PREHOOK: query: CREATE TABLE `customer`( - `c_custkey` bigint, - `c_name` string, - `c_address` string, - `c_city` string, - `c_nation` string, - `c_region` string, - `c_phone` string, - `c_mktsegment` string, - primary key (`c_custkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@customer -POSTHOOK: query: CREATE TABLE `customer`( - `c_custkey` bigint, - `c_name` string, - `c_address` string, - `c_city` string, - `c_nation` string, - `c_region` string, - `c_phone` string, - `c_mktsegment` string, - primary key (`c_custkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@customer -PREHOOK: query: INSERT INTO `customer` -SELECT * FROM `customer_ext` -PREHOOK: type: QUERY -PREHOOK: Input: default@customer_ext -PREHOOK: Output: default@customer -POSTHOOK: query: INSERT INTO `customer` -SELECT * FROM `customer_ext` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@customer_ext -POSTHOOK: Output: default@customer -POSTHOOK: Lineage: customer.c_address SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_address, type:string, comment:null), ] -POSTHOOK: Lineage: customer.c_city SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_city, type:string, comment:null), ] -POSTHOOK: Lineage: customer.c_custkey SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_custkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: customer.c_mktsegment SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_mktsegment, type:string, comment:null), ] -POSTHOOK: Lineage: customer.c_name SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_name, type:string, comment:null), ] -POSTHOOK: Lineage: customer.c_nation SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_nation, type:string, comment:null), ] -POSTHOOK: Lineage: customer.c_phone SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_phone, type:string, comment:null), ] -POSTHOOK: Lineage: customer.c_region SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_region, type:string, comment:null), ] -PREHOOK: query: CREATE TABLE `dates_ext`( - `d_datekey` bigint, - `d_date` string, - `d_dayofweek` string, - `d_month` string, - `d_year` int, - `d_yearmonthnum` int, - `d_yearmonth` string, - `d_daynuminweek` int, - `d_daynuminmonth` int, - `d_daynuminyear` int, - `d_monthnuminyear` int, - `d_weeknuminyear` int, - `d_sellingseason` string, - `d_lastdayinweekfl` int, - `d_lastdayinmonthfl` int, - `d_holidayfl` int , - `d_weekdayfl`int) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dates_ext -POSTHOOK: query: CREATE TABLE `dates_ext`( - `d_datekey` bigint, - `d_date` string, - `d_dayofweek` string, - `d_month` string, - `d_year` int, - `d_yearmonthnum` int, - `d_yearmonth` string, - `d_daynuminweek` int, - `d_daynuminmonth` int, - `d_daynuminyear` int, - `d_monthnuminyear` int, - `d_weeknuminyear` int, - `d_sellingseason` string, - `d_lastdayinweekfl` int, - `d_lastdayinmonthfl` int, - `d_holidayfl` int , - `d_weekdayfl`int) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dates_ext -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/date/' into table `dates_ext` -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@dates_ext -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/date/' into table `dates_ext` -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@dates_ext -PREHOOK: query: CREATE TABLE `dates`( - `d_datekey` bigint, - `d_date` string, - `d_dayofweek` string, - `d_month` string, - `d_year` int, - `d_yearmonthnum` int, - `d_yearmonth` string, - `d_daynuminweek` int, - `d_daynuminmonth` int, - `d_daynuminyear` int, - `d_monthnuminyear` int, - `d_weeknuminyear` int, - `d_sellingseason` string, - `d_lastdayinweekfl` int, - `d_lastdayinmonthfl` int, - `d_holidayfl` int , - `d_weekdayfl`int, - primary key (`d_datekey`) disable rely -) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dates -POSTHOOK: query: CREATE TABLE `dates`( - `d_datekey` bigint, - `d_date` string, - `d_dayofweek` string, - `d_month` string, - `d_year` int, - `d_yearmonthnum` int, - `d_yearmonth` string, - `d_daynuminweek` int, - `d_daynuminmonth` int, - `d_daynuminyear` int, - `d_monthnuminyear` int, - `d_weeknuminyear` int, - `d_sellingseason` string, - `d_lastdayinweekfl` int, - `d_lastdayinmonthfl` int, - `d_holidayfl` int , - `d_weekdayfl`int, - primary key (`d_datekey`) disable rely -) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dates -PREHOOK: query: INSERT INTO `dates` -SELECT * FROM `dates_ext` -PREHOOK: type: QUERY -PREHOOK: Input: default@dates_ext -PREHOOK: Output: default@dates -POSTHOOK: query: INSERT INTO `dates` -SELECT * FROM `dates_ext` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dates_ext -POSTHOOK: Output: default@dates -POSTHOOK: Lineage: dates.d_date SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_date, type:string, comment:null), ] -POSTHOOK: Lineage: dates.d_datekey SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_datekey, type:bigint, comment:null), ] -POSTHOOK: Lineage: dates.d_daynuminmonth SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_daynuminmonth, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_daynuminweek SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_daynuminweek, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_daynuminyear SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_daynuminyear, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_dayofweek SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_dayofweek, type:string, comment:null), ] -POSTHOOK: Lineage: dates.d_holidayfl SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_holidayfl, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_lastdayinmonthfl SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_lastdayinmonthfl, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_lastdayinweekfl SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_lastdayinweekfl, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_month SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_month, type:string, comment:null), ] -POSTHOOK: Lineage: dates.d_monthnuminyear SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_monthnuminyear, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_sellingseason SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_sellingseason, type:string, comment:null), ] -POSTHOOK: Lineage: dates.d_weekdayfl SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_weekdayfl, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_weeknuminyear SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_weeknuminyear, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_year SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_year, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_yearmonth SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_yearmonth, type:string, comment:null), ] -POSTHOOK: Lineage: dates.d_yearmonthnum SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_yearmonthnum, type:int, comment:null), ] -PREHOOK: query: CREATE TABLE `ssb_part_ext`( - `p_partkey` bigint, - `p_name` string, - `p_mfgr` string, - `p_category` string, - `p_brand1` string, - `p_color` string, - `p_type` string, - `p_size` int, - `p_container` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@ssb_part_ext -POSTHOOK: query: CREATE TABLE `ssb_part_ext`( - `p_partkey` bigint, - `p_name` string, - `p_mfgr` string, - `p_category` string, - `p_brand1` string, - `p_color` string, - `p_type` string, - `p_size` int, - `p_container` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ssb_part_ext -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/part/' into table `ssb_part_ext` -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@ssb_part_ext -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/part/' into table `ssb_part_ext` -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@ssb_part_ext -PREHOOK: query: CREATE TABLE `ssb_part`( - `p_partkey` bigint, - `p_name` string, - `p_mfgr` string, - `p_category` string, - `p_brand1` string, - `p_color` string, - `p_type` string, - `p_size` int, - `p_container` string, - primary key (`p_partkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@ssb_part -POSTHOOK: query: CREATE TABLE `ssb_part`( - `p_partkey` bigint, - `p_name` string, - `p_mfgr` string, - `p_category` string, - `p_brand1` string, - `p_color` string, - `p_type` string, - `p_size` int, - `p_container` string, - primary key (`p_partkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ssb_part -PREHOOK: query: INSERT INTO `ssb_part` -SELECT * FROM `ssb_part_ext` -PREHOOK: type: QUERY -PREHOOK: Input: default@ssb_part_ext -PREHOOK: Output: default@ssb_part -POSTHOOK: query: INSERT INTO `ssb_part` -SELECT * FROM `ssb_part_ext` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ssb_part_ext -POSTHOOK: Output: default@ssb_part -POSTHOOK: Lineage: ssb_part.p_brand1 SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_brand1, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_category SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_category, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_color SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_color, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_container SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_container, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_mfgr SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_mfgr, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_name SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_name, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_partkey SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_partkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_size SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_size, type:int, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_type SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_type, type:string, comment:null), ] -PREHOOK: query: CREATE TABLE `supplier_ext`( - `s_suppkey` bigint, - `s_name` string, - `s_address` string, - `s_city` string, - `s_nation` string, - `s_region` string, - `s_phone` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@supplier_ext -POSTHOOK: query: CREATE TABLE `supplier_ext`( - `s_suppkey` bigint, - `s_name` string, - `s_address` string, - `s_city` string, - `s_nation` string, - `s_region` string, - `s_phone` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@supplier_ext -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/supplier/' into table `supplier_ext` -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@supplier_ext -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/supplier/' into table `supplier_ext` -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@supplier_ext -PREHOOK: query: CREATE TABLE `supplier`( - `s_suppkey` bigint, - `s_name` string, - `s_address` string, - `s_city` string, - `s_nation` string, - `s_region` string, - `s_phone` string, - primary key (`s_suppkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@supplier -POSTHOOK: query: CREATE TABLE `supplier`( - `s_suppkey` bigint, - `s_name` string, - `s_address` string, - `s_city` string, - `s_nation` string, - `s_region` string, - `s_phone` string, - primary key (`s_suppkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@supplier -PREHOOK: query: INSERT INTO `supplier` -SELECT * FROM `supplier_ext` -PREHOOK: type: QUERY -PREHOOK: Input: default@supplier_ext -PREHOOK: Output: default@supplier -POSTHOOK: query: INSERT INTO `supplier` -SELECT * FROM `supplier_ext` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@supplier_ext -POSTHOOK: Output: default@supplier -POSTHOOK: Lineage: supplier.s_address SIMPLE [(supplier_ext)supplier_ext.FieldSchema(name:s_address, type:string, comment:null), ] -POSTHOOK: Lineage: supplier.s_city SIMPLE [(supplier_ext)supplier_ext.FieldSchema(name:s_city, type:string, comment:null), ] -POSTHOOK: Lineage: supplier.s_name SIMPLE [(supplier_ext)supplier_ext.FieldSchema(name:s_name, type:string, comment:null), ] -POSTHOOK: Lineage: supplier.s_nation SIMPLE [(supplier_ext)supplier_ext.FieldSchema(name:s_nation, type:string, comment:null), ] -POSTHOOK: Lineage: supplier.s_phone SIMPLE [(supplier_ext)supplier_ext.FieldSchema(name:s_phone, type:string, comment:null), ] -POSTHOOK: Lineage: supplier.s_region SIMPLE [(supplier_ext)supplier_ext.FieldSchema(name:s_region, type:string, comment:null), ] -POSTHOOK: Lineage: supplier.s_suppkey SIMPLE [(supplier_ext)supplier_ext.FieldSchema(name:s_suppkey, type:bigint, comment:null), ] -PREHOOK: query: CREATE TABLE `lineorder_ext`( - `lo_orderkey` bigint, - `lo_linenumber` int, - `lo_custkey` bigint not null disable rely, - `lo_partkey` bigint not null disable rely, - `lo_suppkey` bigint not null disable rely, - `lo_orderdate` bigint not null disable rely, - `lo_ordpriority` string, - `lo_shippriority` string, - `lo_quantity` double, - `lo_extendedprice` double, - `lo_ordtotalprice` double, - `lo_discount` double, - `lo_revenue` double, - `lo_supplycost` double, - `lo_tax` double, - `lo_commitdate` bigint, - `lo_shipmode` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@lineorder_ext -POSTHOOK: query: CREATE TABLE `lineorder_ext`( - `lo_orderkey` bigint, - `lo_linenumber` int, - `lo_custkey` bigint not null disable rely, - `lo_partkey` bigint not null disable rely, - `lo_suppkey` bigint not null disable rely, - `lo_orderdate` bigint not null disable rely, - `lo_ordpriority` string, - `lo_shippriority` string, - `lo_quantity` double, - `lo_extendedprice` double, - `lo_ordtotalprice` double, - `lo_discount` double, - `lo_revenue` double, - `lo_supplycost` double, - `lo_tax` double, - `lo_commitdate` bigint, - `lo_shipmode` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@lineorder_ext -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/lineorder/' into table `lineorder_ext` -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@lineorder_ext -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/lineorder/' into table `lineorder_ext` -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@lineorder_ext -PREHOOK: query: CREATE TABLE `lineorder`( - `lo_orderkey` bigint, - `lo_linenumber` int, - `lo_custkey` bigint not null disable rely, - `lo_partkey` bigint not null disable rely, - `lo_suppkey` bigint not null disable rely, - `lo_orderdate` bigint not null disable rely, - `lo_ordpriority` string, - `lo_shippriority` string, - `lo_quantity` double, - `lo_extendedprice` double, - `lo_ordtotalprice` double, - `lo_discount` double, - `lo_revenue` double, - `lo_supplycost` double, - `lo_tax` double, - `lo_commitdate` bigint, - `lo_shipmode` string, - primary key (`lo_orderkey`) disable rely, - constraint fk1 foreign key (`lo_custkey`) references `customer`(`c_custkey`) disable rely, - constraint fk2 foreign key (`lo_orderdate`) references `dates`(`d_datekey`) disable rely, - constraint fk3 foreign key (`lo_partkey`) references `ssb_part`(`p_partkey`) disable rely, - constraint fk4 foreign key (`lo_suppkey`) references `supplier`(`s_suppkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@lineorder -POSTHOOK: query: CREATE TABLE `lineorder`( - `lo_orderkey` bigint, - `lo_linenumber` int, - `lo_custkey` bigint not null disable rely, - `lo_partkey` bigint not null disable rely, - `lo_suppkey` bigint not null disable rely, - `lo_orderdate` bigint not null disable rely, - `lo_ordpriority` string, - `lo_shippriority` string, - `lo_quantity` double, - `lo_extendedprice` double, - `lo_ordtotalprice` double, - `lo_discount` double, - `lo_revenue` double, - `lo_supplycost` double, - `lo_tax` double, - `lo_commitdate` bigint, - `lo_shipmode` string, - primary key (`lo_orderkey`) disable rely, - constraint fk1 foreign key (`lo_custkey`) references `customer`(`c_custkey`) disable rely, - constraint fk2 foreign key (`lo_orderdate`) references `dates`(`d_datekey`) disable rely, - constraint fk3 foreign key (`lo_partkey`) references `ssb_part`(`p_partkey`) disable rely, - constraint fk4 foreign key (`lo_suppkey`) references `supplier`(`s_suppkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@lineorder -PREHOOK: query: INSERT INTO `lineorder` -SELECT * FROM `lineorder_ext` -PREHOOK: type: QUERY -PREHOOK: Input: default@lineorder_ext -PREHOOK: Output: default@lineorder -POSTHOOK: query: INSERT INTO `lineorder` -SELECT * FROM `lineorder_ext` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@lineorder_ext -POSTHOOK: Output: default@lineorder -POSTHOOK: Lineage: lineorder.lo_commitdate SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_commitdate, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_custkey SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_custkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_discount SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_discount, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_extendedprice SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_extendedprice, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_linenumber SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_linenumber, type:int, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_orderdate SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_orderdate, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_orderkey SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_orderkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_ordpriority SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_ordpriority, type:string, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_ordtotalprice SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_ordtotalprice, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_partkey SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_partkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_quantity SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_quantity, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_revenue SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_revenue, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_shipmode SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_shipmode, type:string, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_shippriority SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_shippriority, type:string, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_suppkey SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_suppkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_supplycost SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_supplycost, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_tax SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_tax, type:double, comment:null), ] -PREHOOK: query: analyze table customer compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@customer -PREHOOK: Output: default@customer -#### A masked pattern was here #### -POSTHOOK: query: analyze table customer compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@customer -POSTHOOK: Output: default@customer -#### A masked pattern was here #### -PREHOOK: query: analyze table dates compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@dates -PREHOOK: Output: default@dates -#### A masked pattern was here #### -POSTHOOK: query: analyze table dates compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dates -POSTHOOK: Output: default@dates -#### A masked pattern was here #### -PREHOOK: query: analyze table ssb_part compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@ssb_part -PREHOOK: Output: default@ssb_part -#### A masked pattern was here #### -POSTHOOK: query: analyze table ssb_part compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ssb_part -POSTHOOK: Output: default@ssb_part -#### A masked pattern was here #### -PREHOOK: query: analyze table supplier compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@supplier -PREHOOK: Output: default@supplier -#### A masked pattern was here #### -POSTHOOK: query: analyze table supplier compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@supplier -POSTHOOK: Output: default@supplier -#### A masked pattern was here #### -PREHOOK: query: analyze table lineorder compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@lineorder -PREHOOK: Output: default@lineorder -#### A masked pattern was here #### -POSTHOOK: query: analyze table lineorder compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@lineorder -POSTHOOK: Output: default@lineorder -#### A masked pattern was here #### -PREHOOK: query: CREATE MATERIALIZED VIEW `ssb_mv` ENABLE REWRITE -AS -SELECT - c_city, - c_nation, - c_region, - d_weeknuminyear, - d_year, - d_yearmonth, - d_yearmonthnum, - lo_discount, - lo_quantity, - p_brand1, - p_category, - p_mfgr, - s_city, - s_nation, - s_region, - lo_revenue, - lo_extendedprice * lo_discount discounted_price, - lo_revenue - lo_supplycost net_revenue -FROM - customer, dates, lineorder, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and lo_custkey = c_custkey -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@customer -PREHOOK: Input: default@dates -PREHOOK: Input: default@lineorder -PREHOOK: Input: default@ssb_part -PREHOOK: Input: default@supplier -PREHOOK: Output: database:default -PREHOOK: Output: default@ssb_mv -POSTHOOK: query: CREATE MATERIALIZED VIEW `ssb_mv` ENABLE REWRITE -AS -SELECT - c_city, - c_nation, - c_region, - d_weeknuminyear, - d_year, - d_yearmonth, - d_yearmonthnum, - lo_discount, - lo_quantity, - p_brand1, - p_category, - p_mfgr, - s_city, - s_nation, - s_region, - lo_revenue, - lo_extendedprice * lo_discount discounted_price, - lo_revenue - lo_supplycost net_revenue -FROM - customer, dates, lineorder, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and lo_custkey = c_custkey -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@customer -POSTHOOK: Input: default@dates -POSTHOOK: Input: default@lineorder -POSTHOOK: Input: default@ssb_part -POSTHOOK: Input: default@supplier -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ssb_mv -PREHOOK: query: explain -select - sum(lo_extendedprice*lo_discount) as revenue -from - lineorder, dates -where - lo_orderdate = d_datekey - and d_year = 1993 - and lo_discount between 1 and 3 - and lo_quantity < 25 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - sum(lo_extendedprice*lo_discount) as revenue -from - lineorder, dates -where - lo_orderdate = d_datekey - and d_year = 1993 - and lo_discount between 1 and 3 - and lo_quantity < 25 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((d_year = 1993) and (lo_quantity < 25.0D) and lo_discount BETWEEN 1.0D AND 3.0D) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: discounted_price (type: double) - outputColumnNames: discounted_price - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(discounted_price) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - sum(lo_extendedprice*lo_discount) as revenue -from - lineorder, dates -where - lo_orderdate = d_datekey - and d_yearmonthnum = 199401 - and lo_discount between 4 and 6 - and lo_quantity between 26 and 35 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - sum(lo_extendedprice*lo_discount) as revenue -from - lineorder, dates -where - lo_orderdate = d_datekey - and d_yearmonthnum = 199401 - and lo_discount between 4 and 6 - and lo_quantity between 26 and 35 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((d_yearmonthnum = 199401) and lo_discount BETWEEN 4.0D AND 6.0D and lo_quantity BETWEEN 26.0D AND 35.0D) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: discounted_price (type: double) - outputColumnNames: discounted_price - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(discounted_price) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - sum(lo_extendedprice*lo_discount) as revenue -from - lineorder, dates -where - lo_orderdate = d_datekey - and d_weeknuminyear = 6 - and d_year = 1994 - and lo_discount between 5 and 7 - and lo_quantity between 26 and 35 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - sum(lo_extendedprice*lo_discount) as revenue -from - lineorder, dates -where - lo_orderdate = d_datekey - and d_weeknuminyear = 6 - and d_year = 1994 - and lo_discount between 5 and 7 - and lo_quantity between 26 and 35 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((d_weeknuminyear = 6) and (d_year = 1994) and lo_discount BETWEEN 5.0D AND 7.0D and lo_quantity BETWEEN 26.0D AND 35.0D) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: discounted_price (type: double) - outputColumnNames: discounted_price - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(discounted_price) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - sum(lo_revenue) as lo_revenue, d_year, p_brand1 -from - lineorder, dates, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and p_category = 'MFGR#12' - and s_region = 'AMERICA' -group by - d_year, p_brand1 -order by - d_year, p_brand1 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - sum(lo_revenue) as lo_revenue, d_year, p_brand1 -from - lineorder, dates, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and p_category = 'MFGR#12' - and s_region = 'AMERICA' -group by - d_year, p_brand1 -order by - d_year, p_brand1 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((p_category = 'MFGR#12') and (s_region = 'AMERICA')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: d_year (type: int), p_brand1 (type: string), lo_revenue (type: double) - outputColumnNames: d_year, p_brand1, lo_revenue - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(lo_revenue) - keys: d_year (type: int), p_brand1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col2 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col2 (type: double), _col0 (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col1 (type: int), _col2 (type: string) - sort order: ++ - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: double) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: double), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - sum(lo_revenue) as lo_revenue, d_year, p_brand1 -from - lineorder, dates, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and p_brand1 between 'MFGR#2221' and 'MFGR#2228' - and s_region = 'ASIA' -group by - d_year, p_brand1 -order by - d_year, p_brand1 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - sum(lo_revenue) as lo_revenue, d_year, p_brand1 -from - lineorder, dates, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and p_brand1 between 'MFGR#2221' and 'MFGR#2228' - and s_region = 'ASIA' -group by - d_year, p_brand1 -order by - d_year, p_brand1 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((s_region = 'ASIA') and p_brand1 BETWEEN 'MFGR#2221' AND 'MFGR#2228') (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: d_year (type: int), p_brand1 (type: string), lo_revenue (type: double) - outputColumnNames: d_year, p_brand1, lo_revenue - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(lo_revenue) - keys: d_year (type: int), p_brand1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col2 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col2 (type: double), _col0 (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col1 (type: int), _col2 (type: string) - sort order: ++ - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: double) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: double), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - sum(lo_revenue) as lo_revenue, d_year, p_brand1 -from - lineorder, dates, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and p_brand1 = 'MFGR#2239' - and s_region = 'EUROPE' -group by - d_year, p_brand1 -order by - d_year, p_brand1 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - sum(lo_revenue) as lo_revenue, d_year, p_brand1 -from - lineorder, dates, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and p_brand1 = 'MFGR#2239' - and s_region = 'EUROPE' -group by - d_year, p_brand1 -order by - d_year, p_brand1 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((p_brand1 = 'MFGR#2239') and (s_region = 'EUROPE')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: d_year (type: int), lo_revenue (type: double) - outputColumnNames: d_year, lo_revenue - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(lo_revenue) - keys: d_year (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col1 (type: double), _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: double) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: double), KEY.reducesinkkey0 (type: int), 'MFGR#2239' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - c_nation, s_nation, d_year, - sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and c_region = 'ASIA' - and s_region = 'ASIA' - and d_year >= 1992 and d_year <= 1997 -group by - c_nation, s_nation, d_year -order by - d_year asc, lo_revenue desc -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - c_nation, s_nation, d_year, - sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and c_region = 'ASIA' - and s_region = 'ASIA' - and d_year >= 1992 and d_year <= 1997 -group by - c_nation, s_nation, d_year -order by - d_year asc, lo_revenue desc -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((c_region = 'ASIA') and (d_year <= 1997) and (d_year >= 1992) and (s_region = 'ASIA')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: c_nation (type: string), d_year (type: int), s_nation (type: string), lo_revenue (type: double) - outputColumnNames: c_nation, d_year, s_nation, lo_revenue - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(lo_revenue) - keys: d_year (type: int), c_nation (type: string), s_nation (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col2 (type: int), _col3 (type: double) - sort order: +- - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - c_city, s_city, d_year, sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and c_nation = 'UNITED STATES' - and s_nation = 'UNITED STATES' - and d_year >= 1992 and d_year <= 1997 -group by - c_city, s_city, d_year -order by - d_year asc, lo_revenue desc -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - c_city, s_city, d_year, sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and c_nation = 'UNITED STATES' - and s_nation = 'UNITED STATES' - and d_year >= 1992 and d_year <= 1997 -group by - c_city, s_city, d_year -order by - d_year asc, lo_revenue desc -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((c_nation = 'UNITED STATES') and (d_year <= 1997) and (d_year >= 1992) and (s_nation = 'UNITED STATES')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: c_city (type: string), d_year (type: int), s_city (type: string), lo_revenue (type: double) - outputColumnNames: c_city, d_year, s_city, lo_revenue - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(lo_revenue) - keys: d_year (type: int), c_city (type: string), s_city (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col2 (type: int), _col3 (type: double) - sort order: +- - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - c_city, s_city, d_year, sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and (c_city='UNITED KI1' or c_city='UNITED KI5') - and (s_city='UNITED KI1' or s_city='UNITED KI5') - and d_year >= 1992 and d_year <= 1997 -group by - c_city, s_city, d_year -order by - d_year asc, lo_revenue desc -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - c_city, s_city, d_year, sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and (c_city='UNITED KI1' or c_city='UNITED KI5') - and (s_city='UNITED KI1' or s_city='UNITED KI5') - and d_year >= 1992 and d_year <= 1997 -group by - c_city, s_city, d_year -order by - d_year asc, lo_revenue desc -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: (((c_city = 'UNITED KI1') or (c_city = 'UNITED KI5')) and ((s_city = 'UNITED KI1') or (s_city = 'UNITED KI5')) and (d_year <= 1997) and (d_year >= 1992)) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(lo_revenue) - keys: d_year (type: int), c_city (type: string), s_city (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col2 (type: int), _col3 (type: double) - sort order: +- - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - c_city, s_city, d_year, sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and (c_city='UNITED KI1' or c_city='UNITED KI5') - and (s_city='UNITED KI1' or s_city='UNITED KI5') - and d_yearmonth = 'Dec1997' -group by - c_city, s_city, d_year -order by - d_year asc, lo_revenue desc -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - c_city, s_city, d_year, sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and (c_city='UNITED KI1' or c_city='UNITED KI5') - and (s_city='UNITED KI1' or s_city='UNITED KI5') - and d_yearmonth = 'Dec1997' -group by - c_city, s_city, d_year -order by - d_year asc, lo_revenue desc -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: (((c_city = 'UNITED KI1') or (c_city = 'UNITED KI5')) and ((s_city = 'UNITED KI1') or (s_city = 'UNITED KI5')) and (d_yearmonth = 'Dec1997')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: c_city (type: string), d_year (type: int), s_city (type: string), lo_revenue (type: double) - outputColumnNames: c_city, d_year, s_city, lo_revenue - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(lo_revenue) - keys: d_year (type: int), c_city (type: string), s_city (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col2 (type: int), _col3 (type: double) - sort order: +- - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - d_year, c_nation, - sum(lo_revenue - lo_supplycost) as profit -from - dates, customer, supplier, ssb_part, lineorder -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_partkey = p_partkey - and lo_orderdate = d_datekey - and c_region = 'AMERICA' - and s_region = 'AMERICA' - and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') -group by - d_year, c_nation -order by - d_year, c_nation -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - d_year, c_nation, - sum(lo_revenue - lo_supplycost) as profit -from - dates, customer, supplier, ssb_part, lineorder -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_partkey = p_partkey - and lo_orderdate = d_datekey - and c_region = 'AMERICA' - and s_region = 'AMERICA' - and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') -group by - d_year, c_nation -order by - d_year, c_nation -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: (((p_mfgr = 'MFGR#1') or (p_mfgr = 'MFGR#2')) and (c_region = 'AMERICA') and (s_region = 'AMERICA')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: c_nation (type: string), d_year (type: int), net_revenue (type: double) - outputColumnNames: c_nation, d_year, net_revenue - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(net_revenue) - keys: d_year (type: int), c_nation (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col2 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col2 (type: double) - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - d_year, s_nation, p_category, - sum(lo_revenue - lo_supplycost) as profit -from - dates, customer, supplier, ssb_part, lineorder -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_partkey = p_partkey - and lo_orderdate = d_datekey - and c_region = 'AMERICA' - and s_region = 'AMERICA' - and (d_year = 1997 or d_year = 1998) - and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') -group by - d_year, s_nation, p_category -order by - d_year, s_nation, p_category -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - d_year, s_nation, p_category, - sum(lo_revenue - lo_supplycost) as profit -from - dates, customer, supplier, ssb_part, lineorder -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_partkey = p_partkey - and lo_orderdate = d_datekey - and c_region = 'AMERICA' - and s_region = 'AMERICA' - and (d_year = 1997 or d_year = 1998) - and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') -group by - d_year, s_nation, p_category -order by - d_year, s_nation, p_category -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: (((d_year = 1997) or (d_year = 1998)) and ((p_mfgr = 'MFGR#1') or (p_mfgr = 'MFGR#2')) and (c_region = 'AMERICA') and (s_region = 'AMERICA')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: d_year (type: int), p_category (type: string), s_nation (type: string), net_revenue (type: double) - outputColumnNames: d_year, p_category, s_nation, net_revenue - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(net_revenue) - keys: d_year (type: int), s_nation (type: string), p_category (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col0 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - d_year, s_city, p_brand1, - sum(lo_revenue - lo_supplycost) as profit -from - dates, customer, supplier, ssb_part, lineorder -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_partkey = p_partkey - and lo_orderdate = d_datekey - and c_region = 'AMERICA' - and s_nation = 'UNITED STATES' - and (d_year = 1997 or d_year = 1998) - and p_category = 'MFGR#14' -group by - d_year, s_city, p_brand1 -order by - d_year, s_city, p_brand1 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - d_year, s_city, p_brand1, - sum(lo_revenue - lo_supplycost) as profit -from - dates, customer, supplier, ssb_part, lineorder -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_partkey = p_partkey - and lo_orderdate = d_datekey - and c_region = 'AMERICA' - and s_nation = 'UNITED STATES' - and (d_year = 1997 or d_year = 1998) - and p_category = 'MFGR#14' -group by - d_year, s_city, p_brand1 -order by - d_year, s_city, p_brand1 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: (((d_year = 1997) or (d_year = 1998)) and (c_region = 'AMERICA') and (p_category = 'MFGR#14') and (s_nation = 'UNITED STATES')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: d_year (type: int), p_brand1 (type: string), s_city (type: string), net_revenue (type: double) - outputColumnNames: d_year, p_brand1, s_city, net_revenue - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(net_revenue) - keys: d_year (type: int), s_city (type: string), p_brand1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col0 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: DROP MATERIALIZED VIEW `ssb_mv` -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@ssb_mv -PREHOOK: Output: default@ssb_mv -POSTHOOK: query: DROP MATERIALIZED VIEW `ssb_mv` -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@ssb_mv -POSTHOOK: Output: default@ssb_mv diff --git a/ql/src/test/results/clientpositive/materialized_view_rewrite_ssb_2.q.out b/ql/src/test/results/clientpositive/materialized_view_rewrite_ssb_2.q.out deleted file mode 100644 index d561208fd8..0000000000 --- a/ql/src/test/results/clientpositive/materialized_view_rewrite_ssb_2.q.out +++ /dev/null @@ -1,1981 +0,0 @@ -PREHOOK: query: CREATE TABLE `customer_ext`( - `c_custkey` bigint, - `c_name` string, - `c_address` string, - `c_city` string, - `c_nation` string, - `c_region` string, - `c_phone` string, - `c_mktsegment` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@customer_ext -POSTHOOK: query: CREATE TABLE `customer_ext`( - `c_custkey` bigint, - `c_name` string, - `c_address` string, - `c_city` string, - `c_nation` string, - `c_region` string, - `c_phone` string, - `c_mktsegment` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@customer_ext -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/customer/' into table `customer_ext` -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@customer_ext -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/customer/' into table `customer_ext` -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@customer_ext -PREHOOK: query: CREATE TABLE `customer`( - `c_custkey` bigint, - `c_name` string, - `c_address` string, - `c_city` string, - `c_nation` string, - `c_region` string, - `c_phone` string, - `c_mktsegment` string, - primary key (`c_custkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@customer -POSTHOOK: query: CREATE TABLE `customer`( - `c_custkey` bigint, - `c_name` string, - `c_address` string, - `c_city` string, - `c_nation` string, - `c_region` string, - `c_phone` string, - `c_mktsegment` string, - primary key (`c_custkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@customer -PREHOOK: query: INSERT INTO `customer` -SELECT * FROM `customer_ext` -PREHOOK: type: QUERY -PREHOOK: Input: default@customer_ext -PREHOOK: Output: default@customer -POSTHOOK: query: INSERT INTO `customer` -SELECT * FROM `customer_ext` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@customer_ext -POSTHOOK: Output: default@customer -POSTHOOK: Lineage: customer.c_address SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_address, type:string, comment:null), ] -POSTHOOK: Lineage: customer.c_city SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_city, type:string, comment:null), ] -POSTHOOK: Lineage: customer.c_custkey SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_custkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: customer.c_mktsegment SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_mktsegment, type:string, comment:null), ] -POSTHOOK: Lineage: customer.c_name SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_name, type:string, comment:null), ] -POSTHOOK: Lineage: customer.c_nation SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_nation, type:string, comment:null), ] -POSTHOOK: Lineage: customer.c_phone SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_phone, type:string, comment:null), ] -POSTHOOK: Lineage: customer.c_region SIMPLE [(customer_ext)customer_ext.FieldSchema(name:c_region, type:string, comment:null), ] -PREHOOK: query: CREATE TABLE `dates_ext`( - `d_datekey` bigint, - `d_date` string, - `d_dayofweek` string, - `d_month` string, - `d_year` int, - `d_yearmonthnum` int, - `d_yearmonth` string, - `d_daynuminweek` int, - `d_daynuminmonth` int, - `d_daynuminyear` int, - `d_monthnuminyear` int, - `d_weeknuminyear` int, - `d_sellingseason` string, - `d_lastdayinweekfl` int, - `d_lastdayinmonthfl` int, - `d_holidayfl` int , - `d_weekdayfl`int) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dates_ext -POSTHOOK: query: CREATE TABLE `dates_ext`( - `d_datekey` bigint, - `d_date` string, - `d_dayofweek` string, - `d_month` string, - `d_year` int, - `d_yearmonthnum` int, - `d_yearmonth` string, - `d_daynuminweek` int, - `d_daynuminmonth` int, - `d_daynuminyear` int, - `d_monthnuminyear` int, - `d_weeknuminyear` int, - `d_sellingseason` string, - `d_lastdayinweekfl` int, - `d_lastdayinmonthfl` int, - `d_holidayfl` int , - `d_weekdayfl`int) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dates_ext -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/date/' into table `dates_ext` -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@dates_ext -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/date/' into table `dates_ext` -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@dates_ext -PREHOOK: query: CREATE TABLE `dates`( - `d_datekey` bigint, - `d_date` string, - `d_dayofweek` string, - `d_month` string, - `d_year` int, - `d_yearmonthnum` int, - `d_yearmonth` string, - `d_daynuminweek` int, - `d_daynuminmonth` int, - `d_daynuminyear` int, - `d_monthnuminyear` int, - `d_weeknuminyear` int, - `d_sellingseason` string, - `d_lastdayinweekfl` int, - `d_lastdayinmonthfl` int, - `d_holidayfl` int , - `d_weekdayfl`int, - primary key (`d_datekey`) disable rely -) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@dates -POSTHOOK: query: CREATE TABLE `dates`( - `d_datekey` bigint, - `d_date` string, - `d_dayofweek` string, - `d_month` string, - `d_year` int, - `d_yearmonthnum` int, - `d_yearmonth` string, - `d_daynuminweek` int, - `d_daynuminmonth` int, - `d_daynuminyear` int, - `d_monthnuminyear` int, - `d_weeknuminyear` int, - `d_sellingseason` string, - `d_lastdayinweekfl` int, - `d_lastdayinmonthfl` int, - `d_holidayfl` int , - `d_weekdayfl`int, - primary key (`d_datekey`) disable rely -) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@dates -PREHOOK: query: INSERT INTO `dates` -SELECT * FROM `dates_ext` -PREHOOK: type: QUERY -PREHOOK: Input: default@dates_ext -PREHOOK: Output: default@dates -POSTHOOK: query: INSERT INTO `dates` -SELECT * FROM `dates_ext` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dates_ext -POSTHOOK: Output: default@dates -POSTHOOK: Lineage: dates.d_date SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_date, type:string, comment:null), ] -POSTHOOK: Lineage: dates.d_datekey SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_datekey, type:bigint, comment:null), ] -POSTHOOK: Lineage: dates.d_daynuminmonth SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_daynuminmonth, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_daynuminweek SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_daynuminweek, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_daynuminyear SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_daynuminyear, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_dayofweek SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_dayofweek, type:string, comment:null), ] -POSTHOOK: Lineage: dates.d_holidayfl SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_holidayfl, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_lastdayinmonthfl SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_lastdayinmonthfl, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_lastdayinweekfl SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_lastdayinweekfl, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_month SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_month, type:string, comment:null), ] -POSTHOOK: Lineage: dates.d_monthnuminyear SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_monthnuminyear, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_sellingseason SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_sellingseason, type:string, comment:null), ] -POSTHOOK: Lineage: dates.d_weekdayfl SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_weekdayfl, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_weeknuminyear SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_weeknuminyear, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_year SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_year, type:int, comment:null), ] -POSTHOOK: Lineage: dates.d_yearmonth SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_yearmonth, type:string, comment:null), ] -POSTHOOK: Lineage: dates.d_yearmonthnum SIMPLE [(dates_ext)dates_ext.FieldSchema(name:d_yearmonthnum, type:int, comment:null), ] -PREHOOK: query: CREATE TABLE `ssb_part_ext`( - `p_partkey` bigint, - `p_name` string, - `p_mfgr` string, - `p_category` string, - `p_brand1` string, - `p_color` string, - `p_type` string, - `p_size` int, - `p_container` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@ssb_part_ext -POSTHOOK: query: CREATE TABLE `ssb_part_ext`( - `p_partkey` bigint, - `p_name` string, - `p_mfgr` string, - `p_category` string, - `p_brand1` string, - `p_color` string, - `p_type` string, - `p_size` int, - `p_container` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ssb_part_ext -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/part/' into table `ssb_part_ext` -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@ssb_part_ext -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/part/' into table `ssb_part_ext` -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@ssb_part_ext -PREHOOK: query: CREATE TABLE `ssb_part`( - `p_partkey` bigint, - `p_name` string, - `p_mfgr` string, - `p_category` string, - `p_brand1` string, - `p_color` string, - `p_type` string, - `p_size` int, - `p_container` string, - primary key (`p_partkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@ssb_part -POSTHOOK: query: CREATE TABLE `ssb_part`( - `p_partkey` bigint, - `p_name` string, - `p_mfgr` string, - `p_category` string, - `p_brand1` string, - `p_color` string, - `p_type` string, - `p_size` int, - `p_container` string, - primary key (`p_partkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ssb_part -PREHOOK: query: INSERT INTO `ssb_part` -SELECT * FROM `ssb_part_ext` -PREHOOK: type: QUERY -PREHOOK: Input: default@ssb_part_ext -PREHOOK: Output: default@ssb_part -POSTHOOK: query: INSERT INTO `ssb_part` -SELECT * FROM `ssb_part_ext` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ssb_part_ext -POSTHOOK: Output: default@ssb_part -POSTHOOK: Lineage: ssb_part.p_brand1 SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_brand1, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_category SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_category, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_color SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_color, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_container SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_container, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_mfgr SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_mfgr, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_name SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_name, type:string, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_partkey SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_partkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_size SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_size, type:int, comment:null), ] -POSTHOOK: Lineage: ssb_part.p_type SIMPLE [(ssb_part_ext)ssb_part_ext.FieldSchema(name:p_type, type:string, comment:null), ] -PREHOOK: query: CREATE TABLE `supplier_ext`( - `s_suppkey` bigint, - `s_name` string, - `s_address` string, - `s_city` string, - `s_nation` string, - `s_region` string, - `s_phone` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@supplier_ext -POSTHOOK: query: CREATE TABLE `supplier_ext`( - `s_suppkey` bigint, - `s_name` string, - `s_address` string, - `s_city` string, - `s_nation` string, - `s_region` string, - `s_phone` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@supplier_ext -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/supplier/' into table `supplier_ext` -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@supplier_ext -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/supplier/' into table `supplier_ext` -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@supplier_ext -PREHOOK: query: CREATE TABLE `supplier`( - `s_suppkey` bigint, - `s_name` string, - `s_address` string, - `s_city` string, - `s_nation` string, - `s_region` string, - `s_phone` string, - primary key (`s_suppkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@supplier -POSTHOOK: query: CREATE TABLE `supplier`( - `s_suppkey` bigint, - `s_name` string, - `s_address` string, - `s_city` string, - `s_nation` string, - `s_region` string, - `s_phone` string, - primary key (`s_suppkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@supplier -PREHOOK: query: INSERT INTO `supplier` -SELECT * FROM `supplier_ext` -PREHOOK: type: QUERY -PREHOOK: Input: default@supplier_ext -PREHOOK: Output: default@supplier -POSTHOOK: query: INSERT INTO `supplier` -SELECT * FROM `supplier_ext` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@supplier_ext -POSTHOOK: Output: default@supplier -POSTHOOK: Lineage: supplier.s_address SIMPLE [(supplier_ext)supplier_ext.FieldSchema(name:s_address, type:string, comment:null), ] -POSTHOOK: Lineage: supplier.s_city SIMPLE [(supplier_ext)supplier_ext.FieldSchema(name:s_city, type:string, comment:null), ] -POSTHOOK: Lineage: supplier.s_name SIMPLE [(supplier_ext)supplier_ext.FieldSchema(name:s_name, type:string, comment:null), ] -POSTHOOK: Lineage: supplier.s_nation SIMPLE [(supplier_ext)supplier_ext.FieldSchema(name:s_nation, type:string, comment:null), ] -POSTHOOK: Lineage: supplier.s_phone SIMPLE [(supplier_ext)supplier_ext.FieldSchema(name:s_phone, type:string, comment:null), ] -POSTHOOK: Lineage: supplier.s_region SIMPLE [(supplier_ext)supplier_ext.FieldSchema(name:s_region, type:string, comment:null), ] -POSTHOOK: Lineage: supplier.s_suppkey SIMPLE [(supplier_ext)supplier_ext.FieldSchema(name:s_suppkey, type:bigint, comment:null), ] -PREHOOK: query: CREATE TABLE `lineorder_ext`( - `lo_orderkey` bigint, - `lo_linenumber` int, - `lo_custkey` bigint not null disable rely, - `lo_partkey` bigint not null disable rely, - `lo_suppkey` bigint not null disable rely, - `lo_orderdate` bigint not null disable rely, - `lo_ordpriority` string, - `lo_shippriority` string, - `lo_quantity` double, - `lo_extendedprice` double, - `lo_ordtotalprice` double, - `lo_discount` double, - `lo_revenue` double, - `lo_supplycost` double, - `lo_tax` double, - `lo_commitdate` bigint, - `lo_shipmode` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@lineorder_ext -POSTHOOK: query: CREATE TABLE `lineorder_ext`( - `lo_orderkey` bigint, - `lo_linenumber` int, - `lo_custkey` bigint not null disable rely, - `lo_partkey` bigint not null disable rely, - `lo_suppkey` bigint not null disable rely, - `lo_orderdate` bigint not null disable rely, - `lo_ordpriority` string, - `lo_shippriority` string, - `lo_quantity` double, - `lo_extendedprice` double, - `lo_ordtotalprice` double, - `lo_discount` double, - `lo_revenue` double, - `lo_supplycost` double, - `lo_tax` double, - `lo_commitdate` bigint, - `lo_shipmode` string) -ROW FORMAT DELIMITED -FIELDS TERMINATED BY '|' -STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@lineorder_ext -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/lineorder/' into table `lineorder_ext` -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@lineorder_ext -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/ssb/lineorder/' into table `lineorder_ext` -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@lineorder_ext -PREHOOK: query: CREATE TABLE `lineorder`( - `lo_orderkey` bigint, - `lo_linenumber` int, - `lo_custkey` bigint not null disable rely, - `lo_partkey` bigint not null disable rely, - `lo_suppkey` bigint not null disable rely, - `lo_orderdate` bigint not null disable rely, - `lo_ordpriority` string, - `lo_shippriority` string, - `lo_quantity` double, - `lo_extendedprice` double, - `lo_ordtotalprice` double, - `lo_discount` double, - `lo_revenue` double, - `lo_supplycost` double, - `lo_tax` double, - `lo_commitdate` bigint, - `lo_shipmode` string, - primary key (`lo_orderkey`) disable rely, - constraint fk1 foreign key (`lo_custkey`) references `customer`(`c_custkey`) disable rely, - constraint fk2 foreign key (`lo_orderdate`) references `dates`(`d_datekey`) disable rely, - constraint fk3 foreign key (`lo_partkey`) references `ssb_part`(`p_partkey`) disable rely, - constraint fk4 foreign key (`lo_suppkey`) references `supplier`(`s_suppkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@lineorder -POSTHOOK: query: CREATE TABLE `lineorder`( - `lo_orderkey` bigint, - `lo_linenumber` int, - `lo_custkey` bigint not null disable rely, - `lo_partkey` bigint not null disable rely, - `lo_suppkey` bigint not null disable rely, - `lo_orderdate` bigint not null disable rely, - `lo_ordpriority` string, - `lo_shippriority` string, - `lo_quantity` double, - `lo_extendedprice` double, - `lo_ordtotalprice` double, - `lo_discount` double, - `lo_revenue` double, - `lo_supplycost` double, - `lo_tax` double, - `lo_commitdate` bigint, - `lo_shipmode` string, - primary key (`lo_orderkey`) disable rely, - constraint fk1 foreign key (`lo_custkey`) references `customer`(`c_custkey`) disable rely, - constraint fk2 foreign key (`lo_orderdate`) references `dates`(`d_datekey`) disable rely, - constraint fk3 foreign key (`lo_partkey`) references `ssb_part`(`p_partkey`) disable rely, - constraint fk4 foreign key (`lo_suppkey`) references `supplier`(`s_suppkey`) disable rely) -STORED AS ORC -TBLPROPERTIES ('transactional'='true') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@lineorder -PREHOOK: query: INSERT INTO `lineorder` -SELECT * FROM `lineorder_ext` -PREHOOK: type: QUERY -PREHOOK: Input: default@lineorder_ext -PREHOOK: Output: default@lineorder -POSTHOOK: query: INSERT INTO `lineorder` -SELECT * FROM `lineorder_ext` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@lineorder_ext -POSTHOOK: Output: default@lineorder -POSTHOOK: Lineage: lineorder.lo_commitdate SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_commitdate, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_custkey SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_custkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_discount SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_discount, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_extendedprice SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_extendedprice, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_linenumber SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_linenumber, type:int, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_orderdate SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_orderdate, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_orderkey SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_orderkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_ordpriority SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_ordpriority, type:string, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_ordtotalprice SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_ordtotalprice, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_partkey SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_partkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_quantity SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_quantity, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_revenue SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_revenue, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_shipmode SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_shipmode, type:string, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_shippriority SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_shippriority, type:string, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_suppkey SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_suppkey, type:bigint, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_supplycost SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_supplycost, type:double, comment:null), ] -POSTHOOK: Lineage: lineorder.lo_tax SIMPLE [(lineorder_ext)lineorder_ext.FieldSchema(name:lo_tax, type:double, comment:null), ] -PREHOOK: query: analyze table customer compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@customer -PREHOOK: Output: default@customer -#### A masked pattern was here #### -POSTHOOK: query: analyze table customer compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@customer -POSTHOOK: Output: default@customer -#### A masked pattern was here #### -PREHOOK: query: analyze table dates compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@dates -PREHOOK: Output: default@dates -#### A masked pattern was here #### -POSTHOOK: query: analyze table dates compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@dates -POSTHOOK: Output: default@dates -#### A masked pattern was here #### -PREHOOK: query: analyze table ssb_part compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@ssb_part -PREHOOK: Output: default@ssb_part -#### A masked pattern was here #### -POSTHOOK: query: analyze table ssb_part compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ssb_part -POSTHOOK: Output: default@ssb_part -#### A masked pattern was here #### -PREHOOK: query: analyze table supplier compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@supplier -PREHOOK: Output: default@supplier -#### A masked pattern was here #### -POSTHOOK: query: analyze table supplier compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@supplier -POSTHOOK: Output: default@supplier -#### A masked pattern was here #### -PREHOOK: query: analyze table lineorder compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@lineorder -PREHOOK: Output: default@lineorder -#### A masked pattern was here #### -POSTHOOK: query: analyze table lineorder compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@lineorder -POSTHOOK: Output: default@lineorder -#### A masked pattern was here #### -PREHOOK: query: CREATE MATERIALIZED VIEW `ssb_mv` ENABLE REWRITE -AS -SELECT - cast(d_year || '-' || d_monthnuminyear || '-' || d_daynuminmonth as timestamp) as `__time`, - c_city, - c_nation, - c_region, - cast(d_weeknuminyear as string) d_weeknuminyear, - cast(d_year as string) d_year, - d_yearmonth, - cast(d_yearmonthnum as string) d_yearmonthnum, - cast(lo_discount as string) lo_discount, - cast(lo_quantity as string) lo_quantity, - p_brand1, - p_category, - p_mfgr, - s_city, - s_nation, - s_region, - lo_revenue, - lo_extendedprice * lo_discount discounted_price, - lo_revenue - lo_supplycost net_revenue -FROM - customer, dates, lineorder, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and lo_custkey = c_custkey -PREHOOK: type: CREATE_MATERIALIZED_VIEW -PREHOOK: Input: default@customer -PREHOOK: Input: default@dates -PREHOOK: Input: default@lineorder -PREHOOK: Input: default@ssb_part -PREHOOK: Input: default@supplier -PREHOOK: Output: database:default -PREHOOK: Output: default@ssb_mv -POSTHOOK: query: CREATE MATERIALIZED VIEW `ssb_mv` ENABLE REWRITE -AS -SELECT - cast(d_year || '-' || d_monthnuminyear || '-' || d_daynuminmonth as timestamp) as `__time`, - c_city, - c_nation, - c_region, - cast(d_weeknuminyear as string) d_weeknuminyear, - cast(d_year as string) d_year, - d_yearmonth, - cast(d_yearmonthnum as string) d_yearmonthnum, - cast(lo_discount as string) lo_discount, - cast(lo_quantity as string) lo_quantity, - p_brand1, - p_category, - p_mfgr, - s_city, - s_nation, - s_region, - lo_revenue, - lo_extendedprice * lo_discount discounted_price, - lo_revenue - lo_supplycost net_revenue -FROM - customer, dates, lineorder, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and lo_custkey = c_custkey -POSTHOOK: type: CREATE_MATERIALIZED_VIEW -POSTHOOK: Input: default@customer -POSTHOOK: Input: default@dates -POSTHOOK: Input: default@lineorder -POSTHOOK: Input: default@ssb_part -POSTHOOK: Input: default@supplier -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ssb_mv -PREHOOK: query: explain -select - sum(lo_extendedprice*lo_discount) as revenue -from - lineorder, dates -where - lo_orderdate = d_datekey - and d_year = 1993 - and lo_discount between 1 and 3 - and lo_quantity < 25 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - sum(lo_extendedprice*lo_discount) as revenue -from - lineorder, dates -where - lo_orderdate = d_datekey - and d_year = 1993 - and lo_discount between 1 and 3 - and lo_quantity < 25 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((UDFToDouble(lo_quantity) < 25.0D) and (UDFToInteger(d_year) = 1993) and UDFToDouble(lo_discount) BETWEEN 1.0D AND 3.0D) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: discounted_price (type: double) - outputColumnNames: discounted_price - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(discounted_price) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - sum(lo_extendedprice*lo_discount) as revenue -from - lineorder, dates -where - lo_orderdate = d_datekey - and d_yearmonthnum = 199401 - and lo_discount between 4 and 6 - and lo_quantity between 26 and 35 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - sum(lo_extendedprice*lo_discount) as revenue -from - lineorder, dates -where - lo_orderdate = d_datekey - and d_yearmonthnum = 199401 - and lo_discount between 4 and 6 - and lo_quantity between 26 and 35 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((UDFToInteger(d_yearmonthnum) = 199401) and UDFToDouble(lo_discount) BETWEEN 4.0D AND 6.0D and UDFToDouble(lo_quantity) BETWEEN 26.0D AND 35.0D) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: discounted_price (type: double) - outputColumnNames: discounted_price - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(discounted_price) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - sum(lo_extendedprice*lo_discount) as revenue -from - lineorder, dates -where - lo_orderdate = d_datekey - and d_weeknuminyear = 6 - and d_year = 1994 - and lo_discount between 5 and 7 - and lo_quantity between 26 and 35 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - sum(lo_extendedprice*lo_discount) as revenue -from - lineorder, dates -where - lo_orderdate = d_datekey - and d_weeknuminyear = 6 - and d_year = 1994 - and lo_discount between 5 and 7 - and lo_quantity between 26 and 35 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((UDFToInteger(d_weeknuminyear) = 6) and (UDFToInteger(d_year) = 1994) and UDFToDouble(lo_discount) BETWEEN 5.0D AND 7.0D and UDFToDouble(lo_quantity) BETWEEN 26.0D AND 35.0D) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: discounted_price (type: double) - outputColumnNames: discounted_price - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(discounted_price) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - sum(lo_revenue) as lo_revenue, d_year, p_brand1 -from - lineorder, dates, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and p_category = 'MFGR#12' - and s_region = 'AMERICA' -group by - d_year, p_brand1 -order by - d_year, p_brand1 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - sum(lo_revenue) as lo_revenue, d_year, p_brand1 -from - lineorder, dates, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and p_category = 'MFGR#12' - and s_region = 'AMERICA' -group by - d_year, p_brand1 -order by - d_year, p_brand1 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((p_category = 'MFGR#12') and (s_region = 'AMERICA')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: UDFToInteger(d_year) (type: int), p_brand1 (type: string), lo_revenue (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(_col2) - keys: _col0 (type: int), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col2 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col2 (type: double), _col0 (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col1 (type: int), _col2 (type: string) - sort order: ++ - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: double) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: double), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - sum(lo_revenue) as lo_revenue, d_year, p_brand1 -from - lineorder, dates, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and p_brand1 between 'MFGR#2221' and 'MFGR#2228' - and s_region = 'ASIA' -group by - d_year, p_brand1 -order by - d_year, p_brand1 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - sum(lo_revenue) as lo_revenue, d_year, p_brand1 -from - lineorder, dates, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and p_brand1 between 'MFGR#2221' and 'MFGR#2228' - and s_region = 'ASIA' -group by - d_year, p_brand1 -order by - d_year, p_brand1 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((s_region = 'ASIA') and p_brand1 BETWEEN 'MFGR#2221' AND 'MFGR#2228') (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: UDFToInteger(d_year) (type: int), p_brand1 (type: string), lo_revenue (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(_col2) - keys: _col0 (type: int), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col2 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col2 (type: double), _col0 (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col1 (type: int), _col2 (type: string) - sort order: ++ - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: double) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: double), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - sum(lo_revenue) as lo_revenue, d_year, p_brand1 -from - lineorder, dates, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and p_brand1 = 'MFGR#2239' - and s_region = 'EUROPE' -group by - d_year, p_brand1 -order by - d_year, p_brand1 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - sum(lo_revenue) as lo_revenue, d_year, p_brand1 -from - lineorder, dates, ssb_part, supplier -where - lo_orderdate = d_datekey - and lo_partkey = p_partkey - and lo_suppkey = s_suppkey - and p_brand1 = 'MFGR#2239' - and s_region = 'EUROPE' -group by - d_year, p_brand1 -order by - d_year, p_brand1 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((p_brand1 = 'MFGR#2239') and (s_region = 'EUROPE')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: UDFToInteger(d_year) (type: int), lo_revenue (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(_col1) - keys: _col0 (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col1 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col1 (type: double), _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: double) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: double), KEY.reducesinkkey0 (type: int), 'MFGR#2239' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - c_nation, s_nation, d_year, - sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and c_region = 'ASIA' - and s_region = 'ASIA' - and d_year >= 1992 and d_year <= 1997 -group by - c_nation, s_nation, d_year -order by - d_year asc, lo_revenue desc -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - c_nation, s_nation, d_year, - sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and c_region = 'ASIA' - and s_region = 'ASIA' - and d_year >= 1992 and d_year <= 1997 -group by - c_nation, s_nation, d_year -order by - d_year asc, lo_revenue desc -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((UDFToInteger(d_year) <= 1997) and (UDFToInteger(d_year) >= 1992) and (c_region = 'ASIA') and (s_region = 'ASIA')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: c_nation (type: string), s_nation (type: string), UDFToInteger(d_year) (type: int), lo_revenue (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(_col3) - keys: _col2 (type: int), _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col2 (type: int), _col3 (type: double) - sort order: +- - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - c_city, s_city, d_year, sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and c_nation = 'UNITED STATES' - and s_nation = 'UNITED STATES' - and d_year >= 1992 and d_year <= 1997 -group by - c_city, s_city, d_year -order by - d_year asc, lo_revenue desc -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - c_city, s_city, d_year, sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and c_nation = 'UNITED STATES' - and s_nation = 'UNITED STATES' - and d_year >= 1992 and d_year <= 1997 -group by - c_city, s_city, d_year -order by - d_year asc, lo_revenue desc -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: ((UDFToInteger(d_year) <= 1997) and (UDFToInteger(d_year) >= 1992) and (c_nation = 'UNITED STATES') and (s_nation = 'UNITED STATES')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: c_city (type: string), s_city (type: string), UDFToInteger(d_year) (type: int), lo_revenue (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(_col3) - keys: _col2 (type: int), _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col2 (type: int), _col3 (type: double) - sort order: +- - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - c_city, s_city, d_year, sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and (c_city='UNITED KI1' or c_city='UNITED KI5') - and (s_city='UNITED KI1' or s_city='UNITED KI5') - and d_year >= 1992 and d_year <= 1997 -group by - c_city, s_city, d_year -order by - d_year asc, lo_revenue desc -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - c_city, s_city, d_year, sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and (c_city='UNITED KI1' or c_city='UNITED KI5') - and (s_city='UNITED KI1' or s_city='UNITED KI5') - and d_year >= 1992 and d_year <= 1997 -group by - c_city, s_city, d_year -order by - d_year asc, lo_revenue desc -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: (((c_city = 'UNITED KI1') or (c_city = 'UNITED KI5')) and ((s_city = 'UNITED KI1') or (s_city = 'UNITED KI5')) and (UDFToInteger(d_year) <= 1997) and (UDFToInteger(d_year) >= 1992)) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: c_city (type: string), s_city (type: string), UDFToInteger(d_year) (type: int), lo_revenue (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(_col3) - keys: _col2 (type: int), _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col2 (type: int), _col3 (type: double) - sort order: +- - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - c_city, s_city, d_year, sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and (c_city='UNITED KI1' or c_city='UNITED KI5') - and (s_city='UNITED KI1' or s_city='UNITED KI5') - and d_yearmonth = 'Dec1997' -group by - c_city, s_city, d_year -order by - d_year asc, lo_revenue desc -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - c_city, s_city, d_year, sum(lo_revenue) as lo_revenue -from - customer, lineorder, supplier, dates -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_orderdate = d_datekey - and (c_city='UNITED KI1' or c_city='UNITED KI5') - and (s_city='UNITED KI1' or s_city='UNITED KI5') - and d_yearmonth = 'Dec1997' -group by - c_city, s_city, d_year -order by - d_year asc, lo_revenue desc -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: (((c_city = 'UNITED KI1') or (c_city = 'UNITED KI5')) and ((s_city = 'UNITED KI1') or (s_city = 'UNITED KI5')) and (d_yearmonth = 'Dec1997')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: c_city (type: string), s_city (type: string), UDFToInteger(d_year) (type: int), lo_revenue (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(_col3) - keys: _col2 (type: int), _col0 (type: string), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col2 (type: int), _col3 (type: double) - sort order: +- - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - d_year, c_nation, - sum(lo_revenue - lo_supplycost) as profit -from - dates, customer, supplier, ssb_part, lineorder -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_partkey = p_partkey - and lo_orderdate = d_datekey - and c_region = 'AMERICA' - and s_region = 'AMERICA' - and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') -group by - d_year, c_nation -order by - d_year, c_nation -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - d_year, c_nation, - sum(lo_revenue - lo_supplycost) as profit -from - dates, customer, supplier, ssb_part, lineorder -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_partkey = p_partkey - and lo_orderdate = d_datekey - and c_region = 'AMERICA' - and s_region = 'AMERICA' - and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') -group by - d_year, c_nation -order by - d_year, c_nation -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: (((p_mfgr = 'MFGR#1') or (p_mfgr = 'MFGR#2')) and (c_region = 'AMERICA') and (s_region = 'AMERICA')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: UDFToInteger(d_year) (type: int), c_nation (type: string), net_revenue (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(_col2) - keys: _col0 (type: int), _col1 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col2 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col2 (type: double) - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: double) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - d_year, s_nation, p_category, - sum(lo_revenue - lo_supplycost) as profit -from - dates, customer, supplier, ssb_part, lineorder -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_partkey = p_partkey - and lo_orderdate = d_datekey - and c_region = 'AMERICA' - and s_region = 'AMERICA' - and (d_year = 1997 or d_year = 1998) - and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') -group by - d_year, s_nation, p_category -order by - d_year, s_nation, p_category -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - d_year, s_nation, p_category, - sum(lo_revenue - lo_supplycost) as profit -from - dates, customer, supplier, ssb_part, lineorder -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_partkey = p_partkey - and lo_orderdate = d_datekey - and c_region = 'AMERICA' - and s_region = 'AMERICA' - and (d_year = 1997 or d_year = 1998) - and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') -group by - d_year, s_nation, p_category -order by - d_year, s_nation, p_category -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: (((UDFToInteger(d_year) = 1997) or (UDFToInteger(d_year) = 1998)) and ((p_mfgr = 'MFGR#1') or (p_mfgr = 'MFGR#2')) and (c_region = 'AMERICA') and (s_region = 'AMERICA')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: UDFToInteger(d_year) (type: int), s_nation (type: string), p_category (type: string), net_revenue (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(_col3) - keys: _col0 (type: int), _col1 (type: string), _col2 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col0 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: explain -select - d_year, s_city, p_brand1, - sum(lo_revenue - lo_supplycost) as profit -from - dates, customer, supplier, ssb_part, lineorder -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_partkey = p_partkey - and lo_orderdate = d_datekey - and c_region = 'AMERICA' - and s_nation = 'UNITED STATES' - and (d_year = 1997 or d_year = 1998) - and p_category = 'MFGR#14' -group by - d_year, s_city, p_brand1 -order by - d_year, s_city, p_brand1 -PREHOOK: type: QUERY -POSTHOOK: query: explain -select - d_year, s_city, p_brand1, - sum(lo_revenue - lo_supplycost) as profit -from - dates, customer, supplier, ssb_part, lineorder -where - lo_custkey = c_custkey - and lo_suppkey = s_suppkey - and lo_partkey = p_partkey - and lo_orderdate = d_datekey - and c_region = 'AMERICA' - and s_nation = 'UNITED STATES' - and (d_year = 1997 or d_year = 1998) - and p_category = 'MFGR#14' -group by - d_year, s_city, p_brand1 -order by - d_year, s_city, p_brand1 -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - -STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: default.ssb_mv - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Filter Operator - predicate: (((UDFToInteger(d_year) = 1997) or (UDFToInteger(d_year) = 1998)) and (c_region = 'AMERICA') and (p_category = 'MFGR#14') and (s_nation = 'UNITED STATES')) (type: boolean) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: UDFToInteger(d_year) (type: int), s_city (type: string), p_brand1 (type: string), net_revenue (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: sum(_col3) - keys: _col0 (type: int), _col1 (type: string), _col2 (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Group By Operator - aggregations: sum(VALUE._col0) - keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - sort order: +++ - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - value expressions: _col3 (type: double) - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col0 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: DROP MATERIALIZED VIEW `ssb_mv` -PREHOOK: type: DROP_MATERIALIZED_VIEW -PREHOOK: Input: default@ssb_mv -PREHOOK: Output: default@ssb_mv -POSTHOOK: query: DROP MATERIALIZED VIEW `ssb_mv` -POSTHOOK: type: DROP_MATERIALIZED_VIEW -POSTHOOK: Input: default@ssb_mv -POSTHOOK: Output: default@ssb_mv diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp index c24055af98..fd52f09fd6 100644 --- a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp +++ b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp @@ -48766,6 +48766,444 @@ uint32_t ThriftHiveMetastore_get_serde_presult::read(::apache::thrift::protocol: return xfer; } + +ThriftHiveMetastore_get_lock_materialization_rebuild_args::~ThriftHiveMetastore_get_lock_materialization_rebuild_args() throw() { +} + + +uint32_t ThriftHiveMetastore_get_lock_materialization_rebuild_args::read(::apache::thrift::protocol::TProtocol* iprot) { + + apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->dbName); + this->__isset.dbName = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->tableName); + this->__isset.tableName = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->txnId); + this->__isset.txnId = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t ThriftHiveMetastore_get_lock_materialization_rebuild_args::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("ThriftHiveMetastore_get_lock_materialization_rebuild_args"); + + xfer += oprot->writeFieldBegin("dbName", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeString(this->dbName); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("tableName", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->tableName); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("txnId", ::apache::thrift::protocol::T_I64, 3); + xfer += oprot->writeI64(this->txnId); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +ThriftHiveMetastore_get_lock_materialization_rebuild_pargs::~ThriftHiveMetastore_get_lock_materialization_rebuild_pargs() throw() { +} + + +uint32_t ThriftHiveMetastore_get_lock_materialization_rebuild_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("ThriftHiveMetastore_get_lock_materialization_rebuild_pargs"); + + xfer += oprot->writeFieldBegin("dbName", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeString((*(this->dbName))); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("tableName", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString((*(this->tableName))); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("txnId", ::apache::thrift::protocol::T_I64, 3); + xfer += oprot->writeI64((*(this->txnId))); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +ThriftHiveMetastore_get_lock_materialization_rebuild_result::~ThriftHiveMetastore_get_lock_materialization_rebuild_result() throw() { +} + + +uint32_t ThriftHiveMetastore_get_lock_materialization_rebuild_result::read(::apache::thrift::protocol::TProtocol* iprot) { + + apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->success.read(iprot); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t ThriftHiveMetastore_get_lock_materialization_rebuild_result::write(::apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("ThriftHiveMetastore_get_lock_materialization_rebuild_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_STRUCT, 0); + xfer += this->success.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +ThriftHiveMetastore_get_lock_materialization_rebuild_presult::~ThriftHiveMetastore_get_lock_materialization_rebuild_presult() throw() { +} + + +uint32_t ThriftHiveMetastore_get_lock_materialization_rebuild_presult::read(::apache::thrift::protocol::TProtocol* iprot) { + + apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += (*(this->success)).read(iprot); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + + +ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args::~ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args() throw() { +} + + +uint32_t ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args::read(::apache::thrift::protocol::TProtocol* iprot) { + + apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->dbName); + this->__isset.dbName = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->tableName); + this->__isset.tableName = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->txnId); + this->__isset.txnId = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args"); + + xfer += oprot->writeFieldBegin("dbName", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeString(this->dbName); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("tableName", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->tableName); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("txnId", ::apache::thrift::protocol::T_I64, 3); + xfer += oprot->writeI64(this->txnId); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_pargs::~ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_pargs() throw() { +} + + +uint32_t ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_pargs"); + + xfer += oprot->writeFieldBegin("dbName", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeString((*(this->dbName))); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("tableName", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString((*(this->tableName))); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("txnId", ::apache::thrift::protocol::T_I64, 3); + xfer += oprot->writeI64((*(this->txnId))); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result::~ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result() throw() { +} + + +uint32_t ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result::read(::apache::thrift::protocol::TProtocol* iprot) { + + apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == ::apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->success); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result::write(::apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_BOOL, 0); + xfer += oprot->writeBool(this->success); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_presult::~ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_presult() throw() { +} + + +uint32_t ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_presult::read(::apache::thrift::protocol::TProtocol* iprot) { + + apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == ::apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool((*(this->success))); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + void ThriftHiveMetastoreClient::getMetaConf(std::string& _return, const std::string& key) { send_getMetaConf(key); @@ -61432,6 +61870,126 @@ void ThriftHiveMetastoreClient::recv_get_serde(SerDeInfo& _return) throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "get_serde failed: unknown result"); } +void ThriftHiveMetastoreClient::get_lock_materialization_rebuild(LockResponse& _return, const std::string& dbName, const std::string& tableName, const int64_t txnId) +{ + send_get_lock_materialization_rebuild(dbName, tableName, txnId); + recv_get_lock_materialization_rebuild(_return); +} + +void ThriftHiveMetastoreClient::send_get_lock_materialization_rebuild(const std::string& dbName, const std::string& tableName, const int64_t txnId) +{ + int32_t cseqid = 0; + oprot_->writeMessageBegin("get_lock_materialization_rebuild", ::apache::thrift::protocol::T_CALL, cseqid); + + ThriftHiveMetastore_get_lock_materialization_rebuild_pargs args; + args.dbName = &dbName; + args.tableName = &tableName; + args.txnId = &txnId; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); +} + +void ThriftHiveMetastoreClient::recv_get_lock_materialization_rebuild(LockResponse& _return) +{ + + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { + ::apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != ::apache::thrift::protocol::T_REPLY) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + if (fname.compare("get_lock_materialization_rebuild") != 0) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + ThriftHiveMetastore_get_lock_materialization_rebuild_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.success) { + // _return pointer has now been filled + return; + } + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "get_lock_materialization_rebuild failed: unknown result"); +} + +bool ThriftHiveMetastoreClient::heartbeat_lock_materialization_rebuild(const std::string& dbName, const std::string& tableName, const int64_t txnId) +{ + send_heartbeat_lock_materialization_rebuild(dbName, tableName, txnId); + return recv_heartbeat_lock_materialization_rebuild(); +} + +void ThriftHiveMetastoreClient::send_heartbeat_lock_materialization_rebuild(const std::string& dbName, const std::string& tableName, const int64_t txnId) +{ + int32_t cseqid = 0; + oprot_->writeMessageBegin("heartbeat_lock_materialization_rebuild", ::apache::thrift::protocol::T_CALL, cseqid); + + ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_pargs args; + args.dbName = &dbName; + args.tableName = &tableName; + args.txnId = &txnId; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); +} + +bool ThriftHiveMetastoreClient::recv_heartbeat_lock_materialization_rebuild() +{ + + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { + ::apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != ::apache::thrift::protocol::T_REPLY) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + if (fname.compare("heartbeat_lock_materialization_rebuild") != 0) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + bool _return; + ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.success) { + return _return; + } + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "heartbeat_lock_materialization_rebuild failed: unknown result"); +} + bool ThriftHiveMetastoreProcessor::dispatchCall(::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, const std::string& fname, int32_t seqid, void* callContext) { ProcessMap::iterator pfn; pfn = processMap_.find(fname); @@ -73350,6 +73908,114 @@ void ThriftHiveMetastoreProcessor::process_get_serde(int32_t seqid, ::apache::th } } +void ThriftHiveMetastoreProcessor::process_get_lock_materialization_rebuild(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) +{ + void* ctx = NULL; + if (this->eventHandler_.get() != NULL) { + ctx = this->eventHandler_->getContext("ThriftHiveMetastore.get_lock_materialization_rebuild", callContext); + } + ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "ThriftHiveMetastore.get_lock_materialization_rebuild"); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preRead(ctx, "ThriftHiveMetastore.get_lock_materialization_rebuild"); + } + + ThriftHiveMetastore_get_lock_materialization_rebuild_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postRead(ctx, "ThriftHiveMetastore.get_lock_materialization_rebuild", bytes); + } + + ThriftHiveMetastore_get_lock_materialization_rebuild_result result; + try { + iface_->get_lock_materialization_rebuild(result.success, args.dbName, args.tableName, args.txnId); + result.__isset.success = true; + } catch (const std::exception& e) { + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->handlerError(ctx, "ThriftHiveMetastore.get_lock_materialization_rebuild"); + } + + ::apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("get_lock_materialization_rebuild", ::apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preWrite(ctx, "ThriftHiveMetastore.get_lock_materialization_rebuild"); + } + + oprot->writeMessageBegin("get_lock_materialization_rebuild", ::apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postWrite(ctx, "ThriftHiveMetastore.get_lock_materialization_rebuild", bytes); + } +} + +void ThriftHiveMetastoreProcessor::process_heartbeat_lock_materialization_rebuild(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) +{ + void* ctx = NULL; + if (this->eventHandler_.get() != NULL) { + ctx = this->eventHandler_->getContext("ThriftHiveMetastore.heartbeat_lock_materialization_rebuild", callContext); + } + ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "ThriftHiveMetastore.heartbeat_lock_materialization_rebuild"); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preRead(ctx, "ThriftHiveMetastore.heartbeat_lock_materialization_rebuild"); + } + + ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postRead(ctx, "ThriftHiveMetastore.heartbeat_lock_materialization_rebuild", bytes); + } + + ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result result; + try { + result.success = iface_->heartbeat_lock_materialization_rebuild(args.dbName, args.tableName, args.txnId); + result.__isset.success = true; + } catch (const std::exception& e) { + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->handlerError(ctx, "ThriftHiveMetastore.heartbeat_lock_materialization_rebuild"); + } + + ::apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("heartbeat_lock_materialization_rebuild", ::apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preWrite(ctx, "ThriftHiveMetastore.heartbeat_lock_materialization_rebuild"); + } + + oprot->writeMessageBegin("heartbeat_lock_materialization_rebuild", ::apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postWrite(ctx, "ThriftHiveMetastore.heartbeat_lock_materialization_rebuild", bytes); + } +} + ::boost::shared_ptr< ::apache::thrift::TProcessor > ThriftHiveMetastoreProcessorFactory::getProcessor(const ::apache::thrift::TConnectionInfo& connInfo) { ::apache::thrift::ReleaseHandler< ThriftHiveMetastoreIfFactory > cleanup(handlerFactory_); ::boost::shared_ptr< ThriftHiveMetastoreIf > handler(handlerFactory_->getHandler(connInfo), cleanup); @@ -90808,7 +91474,283 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_schema_version(SchemaVersion& iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("get_schema_version") != 0) { + if (fname.compare("get_schema_version") != 0) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + // in a bad state, don't commit + using ::apache::thrift::protocol::TProtocolException; + throw TProtocolException(TProtocolException::INVALID_DATA); + } + ThriftHiveMetastore_get_schema_version_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.success) { + // _return pointer has now been filled + sentry.commit(); + return; + } + if (result.__isset.o1) { + sentry.commit(); + throw result.o1; + } + if (result.__isset.o2) { + sentry.commit(); + throw result.o2; + } + // in a bad state, don't commit + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "get_schema_version failed: unknown result"); + } + // seqid != rseqid + this->sync_.updatePending(fname, mtype, rseqid); + + // this will temporarily unlock the readMutex, and let other clients get work done + this->sync_.waitForWork(seqid); + } // end while(true) +} + +void ThriftHiveMetastoreConcurrentClient::get_schema_latest_version(SchemaVersion& _return, const ISchemaName& schemaName) +{ + int32_t seqid = send_get_schema_latest_version(schemaName); + recv_get_schema_latest_version(_return, seqid); +} + +int32_t ThriftHiveMetastoreConcurrentClient::send_get_schema_latest_version(const ISchemaName& schemaName) +{ + int32_t cseqid = this->sync_.generateSeqId(); + ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); + oprot_->writeMessageBegin("get_schema_latest_version", ::apache::thrift::protocol::T_CALL, cseqid); + + ThriftHiveMetastore_get_schema_latest_version_pargs args; + args.schemaName = &schemaName; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + + sentry.commit(); + return cseqid; +} + +void ThriftHiveMetastoreConcurrentClient::recv_get_schema_latest_version(SchemaVersion& _return, const int32_t seqid) +{ + + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; + + // the read mutex gets dropped and reacquired as part of waitForWork() + // The destructor of this sentry wakes up other clients + ::apache::thrift::async::TConcurrentRecvSentry sentry(&this->sync_, seqid); + + while(true) { + if(!this->sync_.getPending(fname, mtype, rseqid)) { + iprot_->readMessageBegin(fname, mtype, rseqid); + } + if(seqid == rseqid) { + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { + ::apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + sentry.commit(); + throw x; + } + if (mtype != ::apache::thrift::protocol::T_REPLY) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + if (fname.compare("get_schema_latest_version") != 0) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + // in a bad state, don't commit + using ::apache::thrift::protocol::TProtocolException; + throw TProtocolException(TProtocolException::INVALID_DATA); + } + ThriftHiveMetastore_get_schema_latest_version_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.success) { + // _return pointer has now been filled + sentry.commit(); + return; + } + if (result.__isset.o1) { + sentry.commit(); + throw result.o1; + } + if (result.__isset.o2) { + sentry.commit(); + throw result.o2; + } + // in a bad state, don't commit + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "get_schema_latest_version failed: unknown result"); + } + // seqid != rseqid + this->sync_.updatePending(fname, mtype, rseqid); + + // this will temporarily unlock the readMutex, and let other clients get work done + this->sync_.waitForWork(seqid); + } // end while(true) +} + +void ThriftHiveMetastoreConcurrentClient::get_schema_all_versions(std::vector & _return, const ISchemaName& schemaName) +{ + int32_t seqid = send_get_schema_all_versions(schemaName); + recv_get_schema_all_versions(_return, seqid); +} + +int32_t ThriftHiveMetastoreConcurrentClient::send_get_schema_all_versions(const ISchemaName& schemaName) +{ + int32_t cseqid = this->sync_.generateSeqId(); + ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); + oprot_->writeMessageBegin("get_schema_all_versions", ::apache::thrift::protocol::T_CALL, cseqid); + + ThriftHiveMetastore_get_schema_all_versions_pargs args; + args.schemaName = &schemaName; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + + sentry.commit(); + return cseqid; +} + +void ThriftHiveMetastoreConcurrentClient::recv_get_schema_all_versions(std::vector & _return, const int32_t seqid) +{ + + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; + + // the read mutex gets dropped and reacquired as part of waitForWork() + // The destructor of this sentry wakes up other clients + ::apache::thrift::async::TConcurrentRecvSentry sentry(&this->sync_, seqid); + + while(true) { + if(!this->sync_.getPending(fname, mtype, rseqid)) { + iprot_->readMessageBegin(fname, mtype, rseqid); + } + if(seqid == rseqid) { + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { + ::apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + sentry.commit(); + throw x; + } + if (mtype != ::apache::thrift::protocol::T_REPLY) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + if (fname.compare("get_schema_all_versions") != 0) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + // in a bad state, don't commit + using ::apache::thrift::protocol::TProtocolException; + throw TProtocolException(TProtocolException::INVALID_DATA); + } + ThriftHiveMetastore_get_schema_all_versions_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.success) { + // _return pointer has now been filled + sentry.commit(); + return; + } + if (result.__isset.o1) { + sentry.commit(); + throw result.o1; + } + if (result.__isset.o2) { + sentry.commit(); + throw result.o2; + } + // in a bad state, don't commit + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "get_schema_all_versions failed: unknown result"); + } + // seqid != rseqid + this->sync_.updatePending(fname, mtype, rseqid); + + // this will temporarily unlock the readMutex, and let other clients get work done + this->sync_.waitForWork(seqid); + } // end while(true) +} + +void ThriftHiveMetastoreConcurrentClient::drop_schema_version(const SchemaVersionDescriptor& schemaVersion) +{ + int32_t seqid = send_drop_schema_version(schemaVersion); + recv_drop_schema_version(seqid); +} + +int32_t ThriftHiveMetastoreConcurrentClient::send_drop_schema_version(const SchemaVersionDescriptor& schemaVersion) +{ + int32_t cseqid = this->sync_.generateSeqId(); + ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); + oprot_->writeMessageBegin("drop_schema_version", ::apache::thrift::protocol::T_CALL, cseqid); + + ThriftHiveMetastore_drop_schema_version_pargs args; + args.schemaVersion = &schemaVersion; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + + sentry.commit(); + return cseqid; +} + +void ThriftHiveMetastoreConcurrentClient::recv_drop_schema_version(const int32_t seqid) +{ + + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; + + // the read mutex gets dropped and reacquired as part of waitForWork() + // The destructor of this sentry wakes up other clients + ::apache::thrift::async::TConcurrentRecvSentry sentry(&this->sync_, seqid); + + while(true) { + if(!this->sync_.getPending(fname, mtype, rseqid)) { + iprot_->readMessageBegin(fname, mtype, rseqid); + } + if(seqid == rseqid) { + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { + ::apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + sentry.commit(); + throw x; + } + if (mtype != ::apache::thrift::protocol::T_REPLY) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + if (fname.compare("drop_schema_version") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); @@ -90817,17 +91759,11 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_schema_version(SchemaVersion& using ::apache::thrift::protocol::TProtocolException; throw TProtocolException(TProtocolException::INVALID_DATA); } - ThriftHiveMetastore_get_schema_version_presult result; - result.success = &_return; + ThriftHiveMetastore_drop_schema_version_presult result; result.read(iprot_); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); - if (result.__isset.success) { - // _return pointer has now been filled - sentry.commit(); - return; - } if (result.__isset.o1) { sentry.commit(); throw result.o1; @@ -90836,8 +91772,8 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_schema_version(SchemaVersion& sentry.commit(); throw result.o2; } - // in a bad state, don't commit - throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "get_schema_version failed: unknown result"); + sentry.commit(); + return; } // seqid != rseqid this->sync_.updatePending(fname, mtype, rseqid); @@ -90847,20 +91783,20 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_schema_version(SchemaVersion& } // end while(true) } -void ThriftHiveMetastoreConcurrentClient::get_schema_latest_version(SchemaVersion& _return, const ISchemaName& schemaName) +void ThriftHiveMetastoreConcurrentClient::get_schemas_by_cols(FindSchemasByColsResp& _return, const FindSchemasByColsRqst& rqst) { - int32_t seqid = send_get_schema_latest_version(schemaName); - recv_get_schema_latest_version(_return, seqid); + int32_t seqid = send_get_schemas_by_cols(rqst); + recv_get_schemas_by_cols(_return, seqid); } -int32_t ThriftHiveMetastoreConcurrentClient::send_get_schema_latest_version(const ISchemaName& schemaName) +int32_t ThriftHiveMetastoreConcurrentClient::send_get_schemas_by_cols(const FindSchemasByColsRqst& rqst) { int32_t cseqid = this->sync_.generateSeqId(); ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); - oprot_->writeMessageBegin("get_schema_latest_version", ::apache::thrift::protocol::T_CALL, cseqid); + oprot_->writeMessageBegin("get_schemas_by_cols", ::apache::thrift::protocol::T_CALL, cseqid); - ThriftHiveMetastore_get_schema_latest_version_pargs args; - args.schemaName = &schemaName; + ThriftHiveMetastore_get_schemas_by_cols_pargs args; + args.rqst = &rqst; args.write(oprot_); oprot_->writeMessageEnd(); @@ -90871,7 +91807,7 @@ int32_t ThriftHiveMetastoreConcurrentClient::send_get_schema_latest_version(cons return cseqid; } -void ThriftHiveMetastoreConcurrentClient::recv_get_schema_latest_version(SchemaVersion& _return, const int32_t seqid) +void ThriftHiveMetastoreConcurrentClient::recv_get_schemas_by_cols(FindSchemasByColsResp& _return, const int32_t seqid) { int32_t rseqid = 0; @@ -90900,7 +91836,7 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_schema_latest_version(SchemaV iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("get_schema_latest_version") != 0) { + if (fname.compare("get_schemas_by_cols") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); @@ -90909,7 +91845,7 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_schema_latest_version(SchemaV using ::apache::thrift::protocol::TProtocolException; throw TProtocolException(TProtocolException::INVALID_DATA); } - ThriftHiveMetastore_get_schema_latest_version_presult result; + ThriftHiveMetastore_get_schemas_by_cols_presult result; result.success = &_return; result.read(iprot_); iprot_->readMessageEnd(); @@ -90924,12 +91860,8 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_schema_latest_version(SchemaV sentry.commit(); throw result.o1; } - if (result.__isset.o2) { - sentry.commit(); - throw result.o2; - } // in a bad state, don't commit - throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "get_schema_latest_version failed: unknown result"); + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "get_schemas_by_cols failed: unknown result"); } // seqid != rseqid this->sync_.updatePending(fname, mtype, rseqid); @@ -90939,20 +91871,20 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_schema_latest_version(SchemaV } // end while(true) } -void ThriftHiveMetastoreConcurrentClient::get_schema_all_versions(std::vector & _return, const ISchemaName& schemaName) +void ThriftHiveMetastoreConcurrentClient::map_schema_version_to_serde(const MapSchemaVersionToSerdeRequest& rqst) { - int32_t seqid = send_get_schema_all_versions(schemaName); - recv_get_schema_all_versions(_return, seqid); + int32_t seqid = send_map_schema_version_to_serde(rqst); + recv_map_schema_version_to_serde(seqid); } -int32_t ThriftHiveMetastoreConcurrentClient::send_get_schema_all_versions(const ISchemaName& schemaName) +int32_t ThriftHiveMetastoreConcurrentClient::send_map_schema_version_to_serde(const MapSchemaVersionToSerdeRequest& rqst) { int32_t cseqid = this->sync_.generateSeqId(); ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); - oprot_->writeMessageBegin("get_schema_all_versions", ::apache::thrift::protocol::T_CALL, cseqid); + oprot_->writeMessageBegin("map_schema_version_to_serde", ::apache::thrift::protocol::T_CALL, cseqid); - ThriftHiveMetastore_get_schema_all_versions_pargs args; - args.schemaName = &schemaName; + ThriftHiveMetastore_map_schema_version_to_serde_pargs args; + args.rqst = &rqst; args.write(oprot_); oprot_->writeMessageEnd(); @@ -90963,7 +91895,7 @@ int32_t ThriftHiveMetastoreConcurrentClient::send_get_schema_all_versions(const return cseqid; } -void ThriftHiveMetastoreConcurrentClient::recv_get_schema_all_versions(std::vector & _return, const int32_t seqid) +void ThriftHiveMetastoreConcurrentClient::recv_map_schema_version_to_serde(const int32_t seqid) { int32_t rseqid = 0; @@ -90992,7 +91924,7 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_schema_all_versions(std::vect iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("get_schema_all_versions") != 0) { + if (fname.compare("map_schema_version_to_serde") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); @@ -91001,17 +91933,11 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_schema_all_versions(std::vect using ::apache::thrift::protocol::TProtocolException; throw TProtocolException(TProtocolException::INVALID_DATA); } - ThriftHiveMetastore_get_schema_all_versions_presult result; - result.success = &_return; + ThriftHiveMetastore_map_schema_version_to_serde_presult result; result.read(iprot_); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); - if (result.__isset.success) { - // _return pointer has now been filled - sentry.commit(); - return; - } if (result.__isset.o1) { sentry.commit(); throw result.o1; @@ -91020,8 +91946,8 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_schema_all_versions(std::vect sentry.commit(); throw result.o2; } - // in a bad state, don't commit - throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "get_schema_all_versions failed: unknown result"); + sentry.commit(); + return; } // seqid != rseqid this->sync_.updatePending(fname, mtype, rseqid); @@ -91031,20 +91957,20 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_schema_all_versions(std::vect } // end while(true) } -void ThriftHiveMetastoreConcurrentClient::drop_schema_version(const SchemaVersionDescriptor& schemaVersion) +void ThriftHiveMetastoreConcurrentClient::set_schema_version_state(const SetSchemaVersionStateRequest& rqst) { - int32_t seqid = send_drop_schema_version(schemaVersion); - recv_drop_schema_version(seqid); + int32_t seqid = send_set_schema_version_state(rqst); + recv_set_schema_version_state(seqid); } -int32_t ThriftHiveMetastoreConcurrentClient::send_drop_schema_version(const SchemaVersionDescriptor& schemaVersion) +int32_t ThriftHiveMetastoreConcurrentClient::send_set_schema_version_state(const SetSchemaVersionStateRequest& rqst) { int32_t cseqid = this->sync_.generateSeqId(); ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); - oprot_->writeMessageBegin("drop_schema_version", ::apache::thrift::protocol::T_CALL, cseqid); + oprot_->writeMessageBegin("set_schema_version_state", ::apache::thrift::protocol::T_CALL, cseqid); - ThriftHiveMetastore_drop_schema_version_pargs args; - args.schemaVersion = &schemaVersion; + ThriftHiveMetastore_set_schema_version_state_pargs args; + args.rqst = &rqst; args.write(oprot_); oprot_->writeMessageEnd(); @@ -91055,7 +91981,7 @@ int32_t ThriftHiveMetastoreConcurrentClient::send_drop_schema_version(const Sche return cseqid; } -void ThriftHiveMetastoreConcurrentClient::recv_drop_schema_version(const int32_t seqid) +void ThriftHiveMetastoreConcurrentClient::recv_set_schema_version_state(const int32_t seqid) { int32_t rseqid = 0; @@ -91084,7 +92010,7 @@ void ThriftHiveMetastoreConcurrentClient::recv_drop_schema_version(const int32_t iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("drop_schema_version") != 0) { + if (fname.compare("set_schema_version_state") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); @@ -91093,7 +92019,7 @@ void ThriftHiveMetastoreConcurrentClient::recv_drop_schema_version(const int32_t using ::apache::thrift::protocol::TProtocolException; throw TProtocolException(TProtocolException::INVALID_DATA); } - ThriftHiveMetastore_drop_schema_version_presult result; + ThriftHiveMetastore_set_schema_version_state_presult result; result.read(iprot_); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); @@ -91106,96 +92032,12 @@ void ThriftHiveMetastoreConcurrentClient::recv_drop_schema_version(const int32_t sentry.commit(); throw result.o2; } - sentry.commit(); - return; - } - // seqid != rseqid - this->sync_.updatePending(fname, mtype, rseqid); - - // this will temporarily unlock the readMutex, and let other clients get work done - this->sync_.waitForWork(seqid); - } // end while(true) -} - -void ThriftHiveMetastoreConcurrentClient::get_schemas_by_cols(FindSchemasByColsResp& _return, const FindSchemasByColsRqst& rqst) -{ - int32_t seqid = send_get_schemas_by_cols(rqst); - recv_get_schemas_by_cols(_return, seqid); -} - -int32_t ThriftHiveMetastoreConcurrentClient::send_get_schemas_by_cols(const FindSchemasByColsRqst& rqst) -{ - int32_t cseqid = this->sync_.generateSeqId(); - ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); - oprot_->writeMessageBegin("get_schemas_by_cols", ::apache::thrift::protocol::T_CALL, cseqid); - - ThriftHiveMetastore_get_schemas_by_cols_pargs args; - args.rqst = &rqst; - args.write(oprot_); - - oprot_->writeMessageEnd(); - oprot_->getTransport()->writeEnd(); - oprot_->getTransport()->flush(); - - sentry.commit(); - return cseqid; -} - -void ThriftHiveMetastoreConcurrentClient::recv_get_schemas_by_cols(FindSchemasByColsResp& _return, const int32_t seqid) -{ - - int32_t rseqid = 0; - std::string fname; - ::apache::thrift::protocol::TMessageType mtype; - - // the read mutex gets dropped and reacquired as part of waitForWork() - // The destructor of this sentry wakes up other clients - ::apache::thrift::async::TConcurrentRecvSentry sentry(&this->sync_, seqid); - - while(true) { - if(!this->sync_.getPending(fname, mtype, rseqid)) { - iprot_->readMessageBegin(fname, mtype, rseqid); - } - if(seqid == rseqid) { - if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { - ::apache::thrift::TApplicationException x; - x.read(iprot_); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - sentry.commit(); - throw x; - } - if (mtype != ::apache::thrift::protocol::T_REPLY) { - iprot_->skip(::apache::thrift::protocol::T_STRUCT); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - } - if (fname.compare("get_schemas_by_cols") != 0) { - iprot_->skip(::apache::thrift::protocol::T_STRUCT); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - - // in a bad state, don't commit - using ::apache::thrift::protocol::TProtocolException; - throw TProtocolException(TProtocolException::INVALID_DATA); - } - ThriftHiveMetastore_get_schemas_by_cols_presult result; - result.success = &_return; - result.read(iprot_); - iprot_->readMessageEnd(); - iprot_->getTransport()->readEnd(); - - if (result.__isset.success) { - // _return pointer has now been filled - sentry.commit(); - return; - } - if (result.__isset.o1) { + if (result.__isset.o3) { sentry.commit(); - throw result.o1; + throw result.o3; } - // in a bad state, don't commit - throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "get_schemas_by_cols failed: unknown result"); + sentry.commit(); + return; } // seqid != rseqid this->sync_.updatePending(fname, mtype, rseqid); @@ -91205,20 +92047,20 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_schemas_by_cols(FindSchemasBy } // end while(true) } -void ThriftHiveMetastoreConcurrentClient::map_schema_version_to_serde(const MapSchemaVersionToSerdeRequest& rqst) +void ThriftHiveMetastoreConcurrentClient::add_serde(const SerDeInfo& serde) { - int32_t seqid = send_map_schema_version_to_serde(rqst); - recv_map_schema_version_to_serde(seqid); + int32_t seqid = send_add_serde(serde); + recv_add_serde(seqid); } -int32_t ThriftHiveMetastoreConcurrentClient::send_map_schema_version_to_serde(const MapSchemaVersionToSerdeRequest& rqst) +int32_t ThriftHiveMetastoreConcurrentClient::send_add_serde(const SerDeInfo& serde) { int32_t cseqid = this->sync_.generateSeqId(); ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); - oprot_->writeMessageBegin("map_schema_version_to_serde", ::apache::thrift::protocol::T_CALL, cseqid); + oprot_->writeMessageBegin("add_serde", ::apache::thrift::protocol::T_CALL, cseqid); - ThriftHiveMetastore_map_schema_version_to_serde_pargs args; - args.rqst = &rqst; + ThriftHiveMetastore_add_serde_pargs args; + args.serde = &serde; args.write(oprot_); oprot_->writeMessageEnd(); @@ -91229,7 +92071,7 @@ int32_t ThriftHiveMetastoreConcurrentClient::send_map_schema_version_to_serde(co return cseqid; } -void ThriftHiveMetastoreConcurrentClient::recv_map_schema_version_to_serde(const int32_t seqid) +void ThriftHiveMetastoreConcurrentClient::recv_add_serde(const int32_t seqid) { int32_t rseqid = 0; @@ -91258,7 +92100,7 @@ void ThriftHiveMetastoreConcurrentClient::recv_map_schema_version_to_serde(const iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("map_schema_version_to_serde") != 0) { + if (fname.compare("add_serde") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); @@ -91267,7 +92109,7 @@ void ThriftHiveMetastoreConcurrentClient::recv_map_schema_version_to_serde(const using ::apache::thrift::protocol::TProtocolException; throw TProtocolException(TProtocolException::INVALID_DATA); } - ThriftHiveMetastore_map_schema_version_to_serde_presult result; + ThriftHiveMetastore_add_serde_presult result; result.read(iprot_); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); @@ -91291,19 +92133,19 @@ void ThriftHiveMetastoreConcurrentClient::recv_map_schema_version_to_serde(const } // end while(true) } -void ThriftHiveMetastoreConcurrentClient::set_schema_version_state(const SetSchemaVersionStateRequest& rqst) +void ThriftHiveMetastoreConcurrentClient::get_serde(SerDeInfo& _return, const GetSerdeRequest& rqst) { - int32_t seqid = send_set_schema_version_state(rqst); - recv_set_schema_version_state(seqid); + int32_t seqid = send_get_serde(rqst); + recv_get_serde(_return, seqid); } -int32_t ThriftHiveMetastoreConcurrentClient::send_set_schema_version_state(const SetSchemaVersionStateRequest& rqst) +int32_t ThriftHiveMetastoreConcurrentClient::send_get_serde(const GetSerdeRequest& rqst) { int32_t cseqid = this->sync_.generateSeqId(); ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); - oprot_->writeMessageBegin("set_schema_version_state", ::apache::thrift::protocol::T_CALL, cseqid); + oprot_->writeMessageBegin("get_serde", ::apache::thrift::protocol::T_CALL, cseqid); - ThriftHiveMetastore_set_schema_version_state_pargs args; + ThriftHiveMetastore_get_serde_pargs args; args.rqst = &rqst; args.write(oprot_); @@ -91315,7 +92157,7 @@ int32_t ThriftHiveMetastoreConcurrentClient::send_set_schema_version_state(const return cseqid; } -void ThriftHiveMetastoreConcurrentClient::recv_set_schema_version_state(const int32_t seqid) +void ThriftHiveMetastoreConcurrentClient::recv_get_serde(SerDeInfo& _return, const int32_t seqid) { int32_t rseqid = 0; @@ -91344,7 +92186,7 @@ void ThriftHiveMetastoreConcurrentClient::recv_set_schema_version_state(const in iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("set_schema_version_state") != 0) { + if (fname.compare("get_serde") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); @@ -91353,11 +92195,17 @@ void ThriftHiveMetastoreConcurrentClient::recv_set_schema_version_state(const in using ::apache::thrift::protocol::TProtocolException; throw TProtocolException(TProtocolException::INVALID_DATA); } - ThriftHiveMetastore_set_schema_version_state_presult result; + ThriftHiveMetastore_get_serde_presult result; + result.success = &_return; result.read(iprot_); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); + if (result.__isset.success) { + // _return pointer has now been filled + sentry.commit(); + return; + } if (result.__isset.o1) { sentry.commit(); throw result.o1; @@ -91366,12 +92214,8 @@ void ThriftHiveMetastoreConcurrentClient::recv_set_schema_version_state(const in sentry.commit(); throw result.o2; } - if (result.__isset.o3) { - sentry.commit(); - throw result.o3; - } - sentry.commit(); - return; + // in a bad state, don't commit + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "get_serde failed: unknown result"); } // seqid != rseqid this->sync_.updatePending(fname, mtype, rseqid); @@ -91381,20 +92225,22 @@ void ThriftHiveMetastoreConcurrentClient::recv_set_schema_version_state(const in } // end while(true) } -void ThriftHiveMetastoreConcurrentClient::add_serde(const SerDeInfo& serde) +void ThriftHiveMetastoreConcurrentClient::get_lock_materialization_rebuild(LockResponse& _return, const std::string& dbName, const std::string& tableName, const int64_t txnId) { - int32_t seqid = send_add_serde(serde); - recv_add_serde(seqid); + int32_t seqid = send_get_lock_materialization_rebuild(dbName, tableName, txnId); + recv_get_lock_materialization_rebuild(_return, seqid); } -int32_t ThriftHiveMetastoreConcurrentClient::send_add_serde(const SerDeInfo& serde) +int32_t ThriftHiveMetastoreConcurrentClient::send_get_lock_materialization_rebuild(const std::string& dbName, const std::string& tableName, const int64_t txnId) { int32_t cseqid = this->sync_.generateSeqId(); ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); - oprot_->writeMessageBegin("add_serde", ::apache::thrift::protocol::T_CALL, cseqid); + oprot_->writeMessageBegin("get_lock_materialization_rebuild", ::apache::thrift::protocol::T_CALL, cseqid); - ThriftHiveMetastore_add_serde_pargs args; - args.serde = &serde; + ThriftHiveMetastore_get_lock_materialization_rebuild_pargs args; + args.dbName = &dbName; + args.tableName = &tableName; + args.txnId = &txnId; args.write(oprot_); oprot_->writeMessageEnd(); @@ -91405,7 +92251,7 @@ int32_t ThriftHiveMetastoreConcurrentClient::send_add_serde(const SerDeInfo& ser return cseqid; } -void ThriftHiveMetastoreConcurrentClient::recv_add_serde(const int32_t seqid) +void ThriftHiveMetastoreConcurrentClient::recv_get_lock_materialization_rebuild(LockResponse& _return, const int32_t seqid) { int32_t rseqid = 0; @@ -91434,7 +92280,7 @@ void ThriftHiveMetastoreConcurrentClient::recv_add_serde(const int32_t seqid) iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("add_serde") != 0) { + if (fname.compare("get_lock_materialization_rebuild") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); @@ -91443,21 +92289,19 @@ void ThriftHiveMetastoreConcurrentClient::recv_add_serde(const int32_t seqid) using ::apache::thrift::protocol::TProtocolException; throw TProtocolException(TProtocolException::INVALID_DATA); } - ThriftHiveMetastore_add_serde_presult result; + ThriftHiveMetastore_get_lock_materialization_rebuild_presult result; + result.success = &_return; result.read(iprot_); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); - if (result.__isset.o1) { - sentry.commit(); - throw result.o1; - } - if (result.__isset.o2) { + if (result.__isset.success) { + // _return pointer has now been filled sentry.commit(); - throw result.o2; + return; } - sentry.commit(); - return; + // in a bad state, don't commit + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "get_lock_materialization_rebuild failed: unknown result"); } // seqid != rseqid this->sync_.updatePending(fname, mtype, rseqid); @@ -91467,20 +92311,22 @@ void ThriftHiveMetastoreConcurrentClient::recv_add_serde(const int32_t seqid) } // end while(true) } -void ThriftHiveMetastoreConcurrentClient::get_serde(SerDeInfo& _return, const GetSerdeRequest& rqst) +bool ThriftHiveMetastoreConcurrentClient::heartbeat_lock_materialization_rebuild(const std::string& dbName, const std::string& tableName, const int64_t txnId) { - int32_t seqid = send_get_serde(rqst); - recv_get_serde(_return, seqid); + int32_t seqid = send_heartbeat_lock_materialization_rebuild(dbName, tableName, txnId); + return recv_heartbeat_lock_materialization_rebuild(seqid); } -int32_t ThriftHiveMetastoreConcurrentClient::send_get_serde(const GetSerdeRequest& rqst) +int32_t ThriftHiveMetastoreConcurrentClient::send_heartbeat_lock_materialization_rebuild(const std::string& dbName, const std::string& tableName, const int64_t txnId) { int32_t cseqid = this->sync_.generateSeqId(); ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); - oprot_->writeMessageBegin("get_serde", ::apache::thrift::protocol::T_CALL, cseqid); + oprot_->writeMessageBegin("heartbeat_lock_materialization_rebuild", ::apache::thrift::protocol::T_CALL, cseqid); - ThriftHiveMetastore_get_serde_pargs args; - args.rqst = &rqst; + ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_pargs args; + args.dbName = &dbName; + args.tableName = &tableName; + args.txnId = &txnId; args.write(oprot_); oprot_->writeMessageEnd(); @@ -91491,7 +92337,7 @@ int32_t ThriftHiveMetastoreConcurrentClient::send_get_serde(const GetSerdeReques return cseqid; } -void ThriftHiveMetastoreConcurrentClient::recv_get_serde(SerDeInfo& _return, const int32_t seqid) +bool ThriftHiveMetastoreConcurrentClient::recv_heartbeat_lock_materialization_rebuild(const int32_t seqid) { int32_t rseqid = 0; @@ -91520,7 +92366,7 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_serde(SerDeInfo& _return, con iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); } - if (fname.compare("get_serde") != 0) { + if (fname.compare("heartbeat_lock_materialization_rebuild") != 0) { iprot_->skip(::apache::thrift::protocol::T_STRUCT); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); @@ -91529,27 +92375,19 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_serde(SerDeInfo& _return, con using ::apache::thrift::protocol::TProtocolException; throw TProtocolException(TProtocolException::INVALID_DATA); } - ThriftHiveMetastore_get_serde_presult result; + bool _return; + ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_presult result; result.success = &_return; result.read(iprot_); iprot_->readMessageEnd(); iprot_->getTransport()->readEnd(); if (result.__isset.success) { - // _return pointer has now been filled sentry.commit(); - return; - } - if (result.__isset.o1) { - sentry.commit(); - throw result.o1; - } - if (result.__isset.o2) { - sentry.commit(); - throw result.o2; + return _return; } // in a bad state, don't commit - throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "get_serde failed: unknown result"); + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "heartbeat_lock_materialization_rebuild failed: unknown result"); } // seqid != rseqid this->sync_.updatePending(fname, mtype, rseqid); diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h index b9e8e24b63..802d8e3fb2 100644 --- a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h +++ b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h @@ -222,6 +222,8 @@ class ThriftHiveMetastoreIf : virtual public ::facebook::fb303::FacebookService virtual void set_schema_version_state(const SetSchemaVersionStateRequest& rqst) = 0; virtual void add_serde(const SerDeInfo& serde) = 0; virtual void get_serde(SerDeInfo& _return, const GetSerdeRequest& rqst) = 0; + virtual void get_lock_materialization_rebuild(LockResponse& _return, const std::string& dbName, const std::string& tableName, const int64_t txnId) = 0; + virtual bool heartbeat_lock_materialization_rebuild(const std::string& dbName, const std::string& tableName, const int64_t txnId) = 0; }; class ThriftHiveMetastoreIfFactory : virtual public ::facebook::fb303::FacebookServiceIfFactory { @@ -878,6 +880,13 @@ class ThriftHiveMetastoreNull : virtual public ThriftHiveMetastoreIf , virtual p void get_serde(SerDeInfo& /* _return */, const GetSerdeRequest& /* rqst */) { return; } + void get_lock_materialization_rebuild(LockResponse& /* _return */, const std::string& /* dbName */, const std::string& /* tableName */, const int64_t /* txnId */) { + return; + } + bool heartbeat_lock_materialization_rebuild(const std::string& /* dbName */, const std::string& /* tableName */, const int64_t /* txnId */) { + bool _return = false; + return _return; + } }; typedef struct _ThriftHiveMetastore_getMetaConf_args__isset { @@ -25399,6 +25408,242 @@ class ThriftHiveMetastore_get_serde_presult { }; +typedef struct _ThriftHiveMetastore_get_lock_materialization_rebuild_args__isset { + _ThriftHiveMetastore_get_lock_materialization_rebuild_args__isset() : dbName(false), tableName(false), txnId(false) {} + bool dbName :1; + bool tableName :1; + bool txnId :1; +} _ThriftHiveMetastore_get_lock_materialization_rebuild_args__isset; + +class ThriftHiveMetastore_get_lock_materialization_rebuild_args { + public: + + ThriftHiveMetastore_get_lock_materialization_rebuild_args(const ThriftHiveMetastore_get_lock_materialization_rebuild_args&); + ThriftHiveMetastore_get_lock_materialization_rebuild_args& operator=(const ThriftHiveMetastore_get_lock_materialization_rebuild_args&); + ThriftHiveMetastore_get_lock_materialization_rebuild_args() : dbName(), tableName(), txnId(0) { + } + + virtual ~ThriftHiveMetastore_get_lock_materialization_rebuild_args() throw(); + std::string dbName; + std::string tableName; + int64_t txnId; + + _ThriftHiveMetastore_get_lock_materialization_rebuild_args__isset __isset; + + void __set_dbName(const std::string& val); + + void __set_tableName(const std::string& val); + + void __set_txnId(const int64_t val); + + bool operator == (const ThriftHiveMetastore_get_lock_materialization_rebuild_args & rhs) const + { + if (!(dbName == rhs.dbName)) + return false; + if (!(tableName == rhs.tableName)) + return false; + if (!(txnId == rhs.txnId)) + return false; + return true; + } + bool operator != (const ThriftHiveMetastore_get_lock_materialization_rebuild_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const ThriftHiveMetastore_get_lock_materialization_rebuild_args & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + + +class ThriftHiveMetastore_get_lock_materialization_rebuild_pargs { + public: + + + virtual ~ThriftHiveMetastore_get_lock_materialization_rebuild_pargs() throw(); + const std::string* dbName; + const std::string* tableName; + const int64_t* txnId; + + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _ThriftHiveMetastore_get_lock_materialization_rebuild_result__isset { + _ThriftHiveMetastore_get_lock_materialization_rebuild_result__isset() : success(false) {} + bool success :1; +} _ThriftHiveMetastore_get_lock_materialization_rebuild_result__isset; + +class ThriftHiveMetastore_get_lock_materialization_rebuild_result { + public: + + ThriftHiveMetastore_get_lock_materialization_rebuild_result(const ThriftHiveMetastore_get_lock_materialization_rebuild_result&); + ThriftHiveMetastore_get_lock_materialization_rebuild_result& operator=(const ThriftHiveMetastore_get_lock_materialization_rebuild_result&); + ThriftHiveMetastore_get_lock_materialization_rebuild_result() { + } + + virtual ~ThriftHiveMetastore_get_lock_materialization_rebuild_result() throw(); + LockResponse success; + + _ThriftHiveMetastore_get_lock_materialization_rebuild_result__isset __isset; + + void __set_success(const LockResponse& val); + + bool operator == (const ThriftHiveMetastore_get_lock_materialization_rebuild_result & rhs) const + { + if (!(success == rhs.success)) + return false; + return true; + } + bool operator != (const ThriftHiveMetastore_get_lock_materialization_rebuild_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const ThriftHiveMetastore_get_lock_materialization_rebuild_result & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _ThriftHiveMetastore_get_lock_materialization_rebuild_presult__isset { + _ThriftHiveMetastore_get_lock_materialization_rebuild_presult__isset() : success(false) {} + bool success :1; +} _ThriftHiveMetastore_get_lock_materialization_rebuild_presult__isset; + +class ThriftHiveMetastore_get_lock_materialization_rebuild_presult { + public: + + + virtual ~ThriftHiveMetastore_get_lock_materialization_rebuild_presult() throw(); + LockResponse* success; + + _ThriftHiveMetastore_get_lock_materialization_rebuild_presult__isset __isset; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + +}; + +typedef struct _ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args__isset { + _ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args__isset() : dbName(false), tableName(false), txnId(false) {} + bool dbName :1; + bool tableName :1; + bool txnId :1; +} _ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args__isset; + +class ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args { + public: + + ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args(const ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args&); + ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args& operator=(const ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args&); + ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args() : dbName(), tableName(), txnId(0) { + } + + virtual ~ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args() throw(); + std::string dbName; + std::string tableName; + int64_t txnId; + + _ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args__isset __isset; + + void __set_dbName(const std::string& val); + + void __set_tableName(const std::string& val); + + void __set_txnId(const int64_t val); + + bool operator == (const ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args & rhs) const + { + if (!(dbName == rhs.dbName)) + return false; + if (!(tableName == rhs.tableName)) + return false; + if (!(txnId == rhs.txnId)) + return false; + return true; + } + bool operator != (const ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + + +class ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_pargs { + public: + + + virtual ~ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_pargs() throw(); + const std::string* dbName; + const std::string* tableName; + const int64_t* txnId; + + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result__isset { + _ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result__isset() : success(false) {} + bool success :1; +} _ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result__isset; + +class ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result { + public: + + ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result(const ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result&); + ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result& operator=(const ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result&); + ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result() : success(0) { + } + + virtual ~ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result() throw(); + bool success; + + _ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result__isset __isset; + + void __set_success(const bool val); + + bool operator == (const ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result & rhs) const + { + if (!(success == rhs.success)) + return false; + return true; + } + bool operator != (const ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_presult__isset { + _ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_presult__isset() : success(false) {} + bool success :1; +} _ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_presult__isset; + +class ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_presult { + public: + + + virtual ~ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_presult() throw(); + bool* success; + + _ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_presult__isset __isset; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + +}; + class ThriftHiveMetastoreClient : virtual public ThriftHiveMetastoreIf, public ::facebook::fb303::FacebookServiceClient { public: ThriftHiveMetastoreClient(boost::shared_ptr< ::apache::thrift::protocol::TProtocol> prot) : @@ -26010,6 +26255,12 @@ class ThriftHiveMetastoreClient : virtual public ThriftHiveMetastoreIf, public void get_serde(SerDeInfo& _return, const GetSerdeRequest& rqst); void send_get_serde(const GetSerdeRequest& rqst); void recv_get_serde(SerDeInfo& _return); + void get_lock_materialization_rebuild(LockResponse& _return, const std::string& dbName, const std::string& tableName, const int64_t txnId); + void send_get_lock_materialization_rebuild(const std::string& dbName, const std::string& tableName, const int64_t txnId); + void recv_get_lock_materialization_rebuild(LockResponse& _return); + bool heartbeat_lock_materialization_rebuild(const std::string& dbName, const std::string& tableName, const int64_t txnId); + void send_heartbeat_lock_materialization_rebuild(const std::string& dbName, const std::string& tableName, const int64_t txnId); + bool recv_heartbeat_lock_materialization_rebuild(); }; class ThriftHiveMetastoreProcessor : public ::facebook::fb303::FacebookServiceProcessor { @@ -26220,6 +26471,8 @@ class ThriftHiveMetastoreProcessor : public ::facebook::fb303::FacebookServiceP void process_set_schema_version_state(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_add_serde(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_get_serde(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); + void process_get_lock_materialization_rebuild(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); + void process_heartbeat_lock_materialization_rebuild(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); public: ThriftHiveMetastoreProcessor(boost::shared_ptr iface) : ::facebook::fb303::FacebookServiceProcessor(iface), @@ -26424,6 +26677,8 @@ class ThriftHiveMetastoreProcessor : public ::facebook::fb303::FacebookServiceP processMap_["set_schema_version_state"] = &ThriftHiveMetastoreProcessor::process_set_schema_version_state; processMap_["add_serde"] = &ThriftHiveMetastoreProcessor::process_add_serde; processMap_["get_serde"] = &ThriftHiveMetastoreProcessor::process_get_serde; + processMap_["get_lock_materialization_rebuild"] = &ThriftHiveMetastoreProcessor::process_get_lock_materialization_rebuild; + processMap_["heartbeat_lock_materialization_rebuild"] = &ThriftHiveMetastoreProcessor::process_heartbeat_lock_materialization_rebuild; } virtual ~ThriftHiveMetastoreProcessor() {} @@ -28380,6 +28635,25 @@ class ThriftHiveMetastoreMultiface : virtual public ThriftHiveMetastoreIf, publi return; } + void get_lock_materialization_rebuild(LockResponse& _return, const std::string& dbName, const std::string& tableName, const int64_t txnId) { + size_t sz = ifaces_.size(); + size_t i = 0; + for (; i < (sz - 1); ++i) { + ifaces_[i]->get_lock_materialization_rebuild(_return, dbName, tableName, txnId); + } + ifaces_[i]->get_lock_materialization_rebuild(_return, dbName, tableName, txnId); + return; + } + + bool heartbeat_lock_materialization_rebuild(const std::string& dbName, const std::string& tableName, const int64_t txnId) { + size_t sz = ifaces_.size(); + size_t i = 0; + for (; i < (sz - 1); ++i) { + ifaces_[i]->heartbeat_lock_materialization_rebuild(dbName, tableName, txnId); + } + return ifaces_[i]->heartbeat_lock_materialization_rebuild(dbName, tableName, txnId); + } + }; // The 'concurrent' client is a thread safe client that correctly handles @@ -28996,6 +29270,12 @@ class ThriftHiveMetastoreConcurrentClient : virtual public ThriftHiveMetastoreIf void get_serde(SerDeInfo& _return, const GetSerdeRequest& rqst); int32_t send_get_serde(const GetSerdeRequest& rqst); void recv_get_serde(SerDeInfo& _return, const int32_t seqid); + void get_lock_materialization_rebuild(LockResponse& _return, const std::string& dbName, const std::string& tableName, const int64_t txnId); + int32_t send_get_lock_materialization_rebuild(const std::string& dbName, const std::string& tableName, const int64_t txnId); + void recv_get_lock_materialization_rebuild(LockResponse& _return, const int32_t seqid); + bool heartbeat_lock_materialization_rebuild(const std::string& dbName, const std::string& tableName, const int64_t txnId); + int32_t send_heartbeat_lock_materialization_rebuild(const std::string& dbName, const std::string& tableName, const int64_t txnId); + bool recv_heartbeat_lock_materialization_rebuild(const int32_t seqid); }; #ifdef _WIN32 diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp index cfec64f96a..c0a39f80e0 100644 --- a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp +++ b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp @@ -1022,6 +1022,16 @@ class ThriftHiveMetastoreHandler : virtual public ThriftHiveMetastoreIf { printf("get_serde\n"); } + void get_lock_materialization_rebuild(LockResponse& _return, const std::string& dbName, const std::string& tableName, const int64_t txnId) { + // Your implementation goes here + printf("get_lock_materialization_rebuild\n"); + } + + bool heartbeat_lock_materialization_rebuild(const std::string& dbName, const std::string& tableName, const int64_t txnId) { + // Your implementation goes here + printf("heartbeat_lock_materialization_rebuild\n"); + } + }; int main(int argc, char **argv) { diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp index 383cb9bc9c..1904047f44 100644 --- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp +++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp @@ -24413,6 +24413,12 @@ __isset.validTxnList = true; void Materialization::__set_invalidationTime(const int64_t val) { this->invalidationTime = val; +__isset.invalidationTime = true; +} + +void Materialization::__set_sourceTablesUpdateDeleteModified(const bool val) { + this->sourceTablesUpdateDeleteModified = val; +__isset.sourceTablesUpdateDeleteModified = true; } uint32_t Materialization::read(::apache::thrift::protocol::TProtocol* iprot) { @@ -24428,7 +24434,6 @@ uint32_t Materialization::read(::apache::thrift::protocol::TProtocol* iprot) { using ::apache::thrift::protocol::TProtocolException; bool isset_tablesUsed = false; - bool isset_invalidationTime = false; while (true) { @@ -24470,7 +24475,15 @@ uint32_t Materialization::read(::apache::thrift::protocol::TProtocol* iprot) { case 3: if (ftype == ::apache::thrift::protocol::T_I64) { xfer += iprot->readI64(this->invalidationTime); - isset_invalidationTime = true; + this->__isset.invalidationTime = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 4: + if (ftype == ::apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->sourceTablesUpdateDeleteModified); + this->__isset.sourceTablesUpdateDeleteModified = true; } else { xfer += iprot->skip(ftype); } @@ -24486,8 +24499,6 @@ uint32_t Materialization::read(::apache::thrift::protocol::TProtocol* iprot) { if (!isset_tablesUsed) throw TProtocolException(TProtocolException::INVALID_DATA); - if (!isset_invalidationTime) - throw TProtocolException(TProtocolException::INVALID_DATA); return xfer; } @@ -24513,10 +24524,16 @@ uint32_t Materialization::write(::apache::thrift::protocol::TProtocol* oprot) co xfer += oprot->writeString(this->validTxnList); xfer += oprot->writeFieldEnd(); } - xfer += oprot->writeFieldBegin("invalidationTime", ::apache::thrift::protocol::T_I64, 3); - xfer += oprot->writeI64(this->invalidationTime); - xfer += oprot->writeFieldEnd(); - + if (this->__isset.invalidationTime) { + xfer += oprot->writeFieldBegin("invalidationTime", ::apache::thrift::protocol::T_I64, 3); + xfer += oprot->writeI64(this->invalidationTime); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.sourceTablesUpdateDeleteModified) { + xfer += oprot->writeFieldBegin("sourceTablesUpdateDeleteModified", ::apache::thrift::protocol::T_BOOL, 4); + xfer += oprot->writeBool(this->sourceTablesUpdateDeleteModified); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -24527,6 +24544,7 @@ void swap(Materialization &a, Materialization &b) { swap(a.tablesUsed, b.tablesUsed); swap(a.validTxnList, b.validTxnList); swap(a.invalidationTime, b.invalidationTime); + swap(a.sourceTablesUpdateDeleteModified, b.sourceTablesUpdateDeleteModified); swap(a.__isset, b.__isset); } @@ -24534,12 +24552,14 @@ Materialization::Materialization(const Materialization& other978) { tablesUsed = other978.tablesUsed; validTxnList = other978.validTxnList; invalidationTime = other978.invalidationTime; + sourceTablesUpdateDeleteModified = other978.sourceTablesUpdateDeleteModified; __isset = other978.__isset; } Materialization& Materialization::operator=(const Materialization& other979) { tablesUsed = other979.tablesUsed; validTxnList = other979.validTxnList; invalidationTime = other979.invalidationTime; + sourceTablesUpdateDeleteModified = other979.sourceTablesUpdateDeleteModified; __isset = other979.__isset; return *this; } @@ -24548,7 +24568,8 @@ void Materialization::printTo(std::ostream& out) const { out << "Materialization("; out << "tablesUsed=" << to_string(tablesUsed); out << ", " << "validTxnList="; (__isset.validTxnList ? (out << to_string(validTxnList)) : (out << "")); - out << ", " << "invalidationTime=" << to_string(invalidationTime); + out << ", " << "invalidationTime="; (__isset.invalidationTime ? (out << to_string(invalidationTime)) : (out << "")); + out << ", " << "sourceTablesUpdateDeleteModified="; (__isset.sourceTablesUpdateDeleteModified ? (out << to_string(sourceTablesUpdateDeleteModified)) : (out << "")); out << ")"; } diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h index 086cfe9db6..55a97de28f 100644 --- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h +++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h @@ -10066,8 +10066,10 @@ inline std::ostream& operator<<(std::ostream& out, const TableMeta& obj) } typedef struct _Materialization__isset { - _Materialization__isset() : validTxnList(false) {} + _Materialization__isset() : validTxnList(false), invalidationTime(false), sourceTablesUpdateDeleteModified(false) {} bool validTxnList :1; + bool invalidationTime :1; + bool sourceTablesUpdateDeleteModified :1; } _Materialization__isset; class Materialization { @@ -10075,13 +10077,14 @@ class Materialization { Materialization(const Materialization&); Materialization& operator=(const Materialization&); - Materialization() : validTxnList(), invalidationTime(0) { + Materialization() : validTxnList(), invalidationTime(0), sourceTablesUpdateDeleteModified(0) { } virtual ~Materialization() throw(); std::set tablesUsed; std::string validTxnList; int64_t invalidationTime; + bool sourceTablesUpdateDeleteModified; _Materialization__isset __isset; @@ -10091,6 +10094,8 @@ class Materialization { void __set_invalidationTime(const int64_t val); + void __set_sourceTablesUpdateDeleteModified(const bool val); + bool operator == (const Materialization & rhs) const { if (!(tablesUsed == rhs.tablesUsed)) @@ -10099,7 +10104,13 @@ class Materialization { return false; else if (__isset.validTxnList && !(validTxnList == rhs.validTxnList)) return false; - if (!(invalidationTime == rhs.invalidationTime)) + if (__isset.invalidationTime != rhs.__isset.invalidationTime) + return false; + else if (__isset.invalidationTime && !(invalidationTime == rhs.invalidationTime)) + return false; + if (__isset.sourceTablesUpdateDeleteModified != rhs.__isset.sourceTablesUpdateDeleteModified) + return false; + else if (__isset.sourceTablesUpdateDeleteModified && !(sourceTablesUpdateDeleteModified == rhs.sourceTablesUpdateDeleteModified)) return false; return true; } diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Materialization.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Materialization.java index 69607280bb..556207ee44 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Materialization.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Materialization.java @@ -41,6 +41,7 @@ private static final org.apache.thrift.protocol.TField TABLES_USED_FIELD_DESC = new org.apache.thrift.protocol.TField("tablesUsed", org.apache.thrift.protocol.TType.SET, (short)1); private static final org.apache.thrift.protocol.TField VALID_TXN_LIST_FIELD_DESC = new org.apache.thrift.protocol.TField("validTxnList", org.apache.thrift.protocol.TType.STRING, (short)2); private static final org.apache.thrift.protocol.TField INVALIDATION_TIME_FIELD_DESC = new org.apache.thrift.protocol.TField("invalidationTime", org.apache.thrift.protocol.TType.I64, (short)3); + private static final org.apache.thrift.protocol.TField SOURCE_TABLES_UPDATE_DELETE_MODIFIED_FIELD_DESC = new org.apache.thrift.protocol.TField("sourceTablesUpdateDeleteModified", org.apache.thrift.protocol.TType.BOOL, (short)4); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -50,13 +51,15 @@ private Set tablesUsed; // required private String validTxnList; // optional - private long invalidationTime; // required + private long invalidationTime; // optional + private boolean sourceTablesUpdateDeleteModified; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { TABLES_USED((short)1, "tablesUsed"), VALID_TXN_LIST((short)2, "validTxnList"), - INVALIDATION_TIME((short)3, "invalidationTime"); + INVALIDATION_TIME((short)3, "invalidationTime"), + SOURCE_TABLES_UPDATE_DELETE_MODIFIED((short)4, "sourceTablesUpdateDeleteModified"); private static final Map byName = new HashMap(); @@ -77,6 +80,8 @@ public static _Fields findByThriftId(int fieldId) { return VALID_TXN_LIST; case 3: // INVALIDATION_TIME return INVALIDATION_TIME; + case 4: // SOURCE_TABLES_UPDATE_DELETE_MODIFIED + return SOURCE_TABLES_UPDATE_DELETE_MODIFIED; default: return null; } @@ -118,8 +123,9 @@ public String getFieldName() { // isset id assignments private static final int __INVALIDATIONTIME_ISSET_ID = 0; + private static final int __SOURCETABLESUPDATEDELETEMODIFIED_ISSET_ID = 1; private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.VALID_TXN_LIST}; + private static final _Fields optionals[] = {_Fields.VALID_TXN_LIST,_Fields.INVALIDATION_TIME,_Fields.SOURCE_TABLES_UPDATE_DELETE_MODIFIED}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -128,8 +134,10 @@ public String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); tmpMap.put(_Fields.VALID_TXN_LIST, new org.apache.thrift.meta_data.FieldMetaData("validTxnList", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); - tmpMap.put(_Fields.INVALIDATION_TIME, new org.apache.thrift.meta_data.FieldMetaData("invalidationTime", org.apache.thrift.TFieldRequirementType.REQUIRED, + tmpMap.put(_Fields.INVALIDATION_TIME, new org.apache.thrift.meta_data.FieldMetaData("invalidationTime", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.SOURCE_TABLES_UPDATE_DELETE_MODIFIED, new org.apache.thrift.meta_data.FieldMetaData("sourceTablesUpdateDeleteModified", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(Materialization.class, metaDataMap); } @@ -138,13 +146,10 @@ public Materialization() { } public Materialization( - Set tablesUsed, - long invalidationTime) + Set tablesUsed) { this(); this.tablesUsed = tablesUsed; - this.invalidationTime = invalidationTime; - setInvalidationTimeIsSet(true); } /** @@ -160,6 +165,7 @@ public Materialization(Materialization other) { this.validTxnList = other.validTxnList; } this.invalidationTime = other.invalidationTime; + this.sourceTablesUpdateDeleteModified = other.sourceTablesUpdateDeleteModified; } public Materialization deepCopy() { @@ -172,6 +178,8 @@ public void clear() { this.validTxnList = null; setInvalidationTimeIsSet(false); this.invalidationTime = 0; + setSourceTablesUpdateDeleteModifiedIsSet(false); + this.sourceTablesUpdateDeleteModified = false; } public int getTablesUsedSize() { @@ -257,6 +265,28 @@ public void setInvalidationTimeIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __INVALIDATIONTIME_ISSET_ID, value); } + public boolean isSourceTablesUpdateDeleteModified() { + return this.sourceTablesUpdateDeleteModified; + } + + public void setSourceTablesUpdateDeleteModified(boolean sourceTablesUpdateDeleteModified) { + this.sourceTablesUpdateDeleteModified = sourceTablesUpdateDeleteModified; + setSourceTablesUpdateDeleteModifiedIsSet(true); + } + + public void unsetSourceTablesUpdateDeleteModified() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __SOURCETABLESUPDATEDELETEMODIFIED_ISSET_ID); + } + + /** Returns true if field sourceTablesUpdateDeleteModified is set (has been assigned a value) and false otherwise */ + public boolean isSetSourceTablesUpdateDeleteModified() { + return EncodingUtils.testBit(__isset_bitfield, __SOURCETABLESUPDATEDELETEMODIFIED_ISSET_ID); + } + + public void setSourceTablesUpdateDeleteModifiedIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __SOURCETABLESUPDATEDELETEMODIFIED_ISSET_ID, value); + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case TABLES_USED: @@ -283,6 +313,14 @@ public void setFieldValue(_Fields field, Object value) { } break; + case SOURCE_TABLES_UPDATE_DELETE_MODIFIED: + if (value == null) { + unsetSourceTablesUpdateDeleteModified(); + } else { + setSourceTablesUpdateDeleteModified((Boolean)value); + } + break; + } } @@ -297,6 +335,9 @@ public Object getFieldValue(_Fields field) { case INVALIDATION_TIME: return getInvalidationTime(); + case SOURCE_TABLES_UPDATE_DELETE_MODIFIED: + return isSourceTablesUpdateDeleteModified(); + } throw new IllegalStateException(); } @@ -314,6 +355,8 @@ public boolean isSet(_Fields field) { return isSetValidTxnList(); case INVALIDATION_TIME: return isSetInvalidationTime(); + case SOURCE_TABLES_UPDATE_DELETE_MODIFIED: + return isSetSourceTablesUpdateDeleteModified(); } throw new IllegalStateException(); } @@ -349,8 +392,8 @@ public boolean equals(Materialization that) { return false; } - boolean this_present_invalidationTime = true; - boolean that_present_invalidationTime = true; + boolean this_present_invalidationTime = true && this.isSetInvalidationTime(); + boolean that_present_invalidationTime = true && that.isSetInvalidationTime(); if (this_present_invalidationTime || that_present_invalidationTime) { if (!(this_present_invalidationTime && that_present_invalidationTime)) return false; @@ -358,6 +401,15 @@ public boolean equals(Materialization that) { return false; } + boolean this_present_sourceTablesUpdateDeleteModified = true && this.isSetSourceTablesUpdateDeleteModified(); + boolean that_present_sourceTablesUpdateDeleteModified = true && that.isSetSourceTablesUpdateDeleteModified(); + if (this_present_sourceTablesUpdateDeleteModified || that_present_sourceTablesUpdateDeleteModified) { + if (!(this_present_sourceTablesUpdateDeleteModified && that_present_sourceTablesUpdateDeleteModified)) + return false; + if (this.sourceTablesUpdateDeleteModified != that.sourceTablesUpdateDeleteModified) + return false; + } + return true; } @@ -375,11 +427,16 @@ public int hashCode() { if (present_validTxnList) list.add(validTxnList); - boolean present_invalidationTime = true; + boolean present_invalidationTime = true && (isSetInvalidationTime()); list.add(present_invalidationTime); if (present_invalidationTime) list.add(invalidationTime); + boolean present_sourceTablesUpdateDeleteModified = true && (isSetSourceTablesUpdateDeleteModified()); + list.add(present_sourceTablesUpdateDeleteModified); + if (present_sourceTablesUpdateDeleteModified) + list.add(sourceTablesUpdateDeleteModified); + return list.hashCode(); } @@ -421,6 +478,16 @@ public int compareTo(Materialization other) { return lastComparison; } } + lastComparison = Boolean.valueOf(isSetSourceTablesUpdateDeleteModified()).compareTo(other.isSetSourceTablesUpdateDeleteModified()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSourceTablesUpdateDeleteModified()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sourceTablesUpdateDeleteModified, other.sourceTablesUpdateDeleteModified); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -458,10 +525,18 @@ public String toString() { } first = false; } - if (!first) sb.append(", "); - sb.append("invalidationTime:"); - sb.append(this.invalidationTime); - first = false; + if (isSetInvalidationTime()) { + if (!first) sb.append(", "); + sb.append("invalidationTime:"); + sb.append(this.invalidationTime); + first = false; + } + if (isSetSourceTablesUpdateDeleteModified()) { + if (!first) sb.append(", "); + sb.append("sourceTablesUpdateDeleteModified:"); + sb.append(this.sourceTablesUpdateDeleteModified); + first = false; + } sb.append(")"); return sb.toString(); } @@ -472,10 +547,6 @@ public void validate() throws org.apache.thrift.TException { throw new org.apache.thrift.protocol.TProtocolException("Required field 'tablesUsed' is unset! Struct:" + toString()); } - if (!isSetInvalidationTime()) { - throw new org.apache.thrift.protocol.TProtocolException("Required field 'invalidationTime' is unset! Struct:" + toString()); - } - // check for sub-struct validity } @@ -549,6 +620,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Materialization str org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 4: // SOURCE_TABLES_UPDATE_DELETE_MODIFIED + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.sourceTablesUpdateDeleteModified = iprot.readBool(); + struct.setSourceTablesUpdateDeleteModifiedIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -581,9 +660,16 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, Materialization st oprot.writeFieldEnd(); } } - oprot.writeFieldBegin(INVALIDATION_TIME_FIELD_DESC); - oprot.writeI64(struct.invalidationTime); - oprot.writeFieldEnd(); + if (struct.isSetInvalidationTime()) { + oprot.writeFieldBegin(INVALIDATION_TIME_FIELD_DESC); + oprot.writeI64(struct.invalidationTime); + oprot.writeFieldEnd(); + } + if (struct.isSetSourceTablesUpdateDeleteModified()) { + oprot.writeFieldBegin(SOURCE_TABLES_UPDATE_DELETE_MODIFIED_FIELD_DESC); + oprot.writeBool(struct.sourceTablesUpdateDeleteModified); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -608,15 +694,26 @@ public void write(org.apache.thrift.protocol.TProtocol prot, Materialization str oprot.writeString(_iter828); } } - oprot.writeI64(struct.invalidationTime); BitSet optionals = new BitSet(); if (struct.isSetValidTxnList()) { optionals.set(0); } - oprot.writeBitSet(optionals, 1); + if (struct.isSetInvalidationTime()) { + optionals.set(1); + } + if (struct.isSetSourceTablesUpdateDeleteModified()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); if (struct.isSetValidTxnList()) { oprot.writeString(struct.validTxnList); } + if (struct.isSetInvalidationTime()) { + oprot.writeI64(struct.invalidationTime); + } + if (struct.isSetSourceTablesUpdateDeleteModified()) { + oprot.writeBool(struct.sourceTablesUpdateDeleteModified); + } } @Override @@ -633,13 +730,19 @@ public void read(org.apache.thrift.protocol.TProtocol prot, Materialization stru } } struct.setTablesUsedIsSet(true); - struct.invalidationTime = iprot.readI64(); - struct.setInvalidationTimeIsSet(true); - BitSet incoming = iprot.readBitSet(1); + BitSet incoming = iprot.readBitSet(3); if (incoming.get(0)) { struct.validTxnList = iprot.readString(); struct.setValidTxnListIsSet(true); } + if (incoming.get(1)) { + struct.invalidationTime = iprot.readI64(); + struct.setInvalidationTimeIsSet(true); + } + if (incoming.get(2)) { + struct.sourceTablesUpdateDeleteModified = iprot.readBool(); + struct.setSourceTablesUpdateDeleteModifiedIsSet(true); + } } } diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java index 0bca4ebafd..afe82e3a00 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java @@ -442,6 +442,10 @@ public SerDeInfo get_serde(GetSerdeRequest rqst) throws NoSuchObjectException, MetaException, org.apache.thrift.TException; + public LockResponse get_lock_materialization_rebuild(String dbName, String tableName, long txnId) throws org.apache.thrift.TException; + + public boolean heartbeat_lock_materialization_rebuild(String dbName, String tableName, long txnId) throws org.apache.thrift.TException; + } @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public interface AsyncIface extends com.facebook.fb303.FacebookService .AsyncIface { @@ -846,6 +850,10 @@ public void get_serde(GetSerdeRequest rqst, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + public void get_lock_materialization_rebuild(String dbName, String tableName, long txnId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void heartbeat_lock_materialization_rebuild(String dbName, String tableName, long txnId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + } @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class Client extends com.facebook.fb303.FacebookService.Client implements Iface { @@ -6634,6 +6642,56 @@ public SerDeInfo recv_get_serde() throws NoSuchObjectException, MetaException, o throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "get_serde failed: unknown result"); } + public LockResponse get_lock_materialization_rebuild(String dbName, String tableName, long txnId) throws org.apache.thrift.TException + { + send_get_lock_materialization_rebuild(dbName, tableName, txnId); + return recv_get_lock_materialization_rebuild(); + } + + public void send_get_lock_materialization_rebuild(String dbName, String tableName, long txnId) throws org.apache.thrift.TException + { + get_lock_materialization_rebuild_args args = new get_lock_materialization_rebuild_args(); + args.setDbName(dbName); + args.setTableName(tableName); + args.setTxnId(txnId); + sendBase("get_lock_materialization_rebuild", args); + } + + public LockResponse recv_get_lock_materialization_rebuild() throws org.apache.thrift.TException + { + get_lock_materialization_rebuild_result result = new get_lock_materialization_rebuild_result(); + receiveBase(result, "get_lock_materialization_rebuild"); + if (result.isSetSuccess()) { + return result.success; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "get_lock_materialization_rebuild failed: unknown result"); + } + + public boolean heartbeat_lock_materialization_rebuild(String dbName, String tableName, long txnId) throws org.apache.thrift.TException + { + send_heartbeat_lock_materialization_rebuild(dbName, tableName, txnId); + return recv_heartbeat_lock_materialization_rebuild(); + } + + public void send_heartbeat_lock_materialization_rebuild(String dbName, String tableName, long txnId) throws org.apache.thrift.TException + { + heartbeat_lock_materialization_rebuild_args args = new heartbeat_lock_materialization_rebuild_args(); + args.setDbName(dbName); + args.setTableName(tableName); + args.setTxnId(txnId); + sendBase("heartbeat_lock_materialization_rebuild", args); + } + + public boolean recv_heartbeat_lock_materialization_rebuild() throws org.apache.thrift.TException + { + heartbeat_lock_materialization_rebuild_result result = new heartbeat_lock_materialization_rebuild_result(); + receiveBase(result, "heartbeat_lock_materialization_rebuild"); + if (result.isSetSuccess()) { + return result.success; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "heartbeat_lock_materialization_rebuild failed: unknown result"); + } + } @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class AsyncClient extends com.facebook.fb303.FacebookService.AsyncClient implements AsyncIface { @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class Factory implements org.apache.thrift.async.TAsyncClientFactory { @@ -13526,6 +13584,82 @@ public SerDeInfo getResult() throws NoSuchObjectException, MetaException, org.ap } } + public void get_lock_materialization_rebuild(String dbName, String tableName, long txnId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + get_lock_materialization_rebuild_call method_call = new get_lock_materialization_rebuild_call(dbName, tableName, txnId, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class get_lock_materialization_rebuild_call extends org.apache.thrift.async.TAsyncMethodCall { + private String dbName; + private String tableName; + private long txnId; + public get_lock_materialization_rebuild_call(String dbName, String tableName, long txnId, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.dbName = dbName; + this.tableName = tableName; + this.txnId = txnId; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("get_lock_materialization_rebuild", org.apache.thrift.protocol.TMessageType.CALL, 0)); + get_lock_materialization_rebuild_args args = new get_lock_materialization_rebuild_args(); + args.setDbName(dbName); + args.setTableName(tableName); + args.setTxnId(txnId); + args.write(prot); + prot.writeMessageEnd(); + } + + public LockResponse getResult() throws org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_get_lock_materialization_rebuild(); + } + } + + public void heartbeat_lock_materialization_rebuild(String dbName, String tableName, long txnId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + heartbeat_lock_materialization_rebuild_call method_call = new heartbeat_lock_materialization_rebuild_call(dbName, tableName, txnId, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class heartbeat_lock_materialization_rebuild_call extends org.apache.thrift.async.TAsyncMethodCall { + private String dbName; + private String tableName; + private long txnId; + public heartbeat_lock_materialization_rebuild_call(String dbName, String tableName, long txnId, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.dbName = dbName; + this.tableName = tableName; + this.txnId = txnId; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("heartbeat_lock_materialization_rebuild", org.apache.thrift.protocol.TMessageType.CALL, 0)); + heartbeat_lock_materialization_rebuild_args args = new heartbeat_lock_materialization_rebuild_args(); + args.setDbName(dbName); + args.setTableName(tableName); + args.setTxnId(txnId); + args.write(prot); + prot.writeMessageEnd(); + } + + public boolean getResult() throws org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_heartbeat_lock_materialization_rebuild(); + } + } + } @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class Processor extends com.facebook.fb303.FacebookService.Processor implements org.apache.thrift.TProcessor { @@ -13739,6 +13873,8 @@ protected Processor(I iface, Map extends org.apache.thrift.ProcessFunction { + public get_lock_materialization_rebuild() { + super("get_lock_materialization_rebuild"); + } + + public get_lock_materialization_rebuild_args getEmptyArgsInstance() { + return new get_lock_materialization_rebuild_args(); + } + + protected boolean isOneway() { + return false; + } + + public get_lock_materialization_rebuild_result getResult(I iface, get_lock_materialization_rebuild_args args) throws org.apache.thrift.TException { + get_lock_materialization_rebuild_result result = new get_lock_materialization_rebuild_result(); + result.success = iface.get_lock_materialization_rebuild(args.dbName, args.tableName, args.txnId); + return result; + } + } + + @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class heartbeat_lock_materialization_rebuild extends org.apache.thrift.ProcessFunction { + public heartbeat_lock_materialization_rebuild() { + super("heartbeat_lock_materialization_rebuild"); + } + + public heartbeat_lock_materialization_rebuild_args getEmptyArgsInstance() { + return new heartbeat_lock_materialization_rebuild_args(); + } + + protected boolean isOneway() { + return false; + } + + public heartbeat_lock_materialization_rebuild_result getResult(I iface, heartbeat_lock_materialization_rebuild_args args) throws org.apache.thrift.TException { + heartbeat_lock_materialization_rebuild_result result = new heartbeat_lock_materialization_rebuild_result(); + result.success = iface.heartbeat_lock_materialization_rebuild(args.dbName, args.tableName, args.txnId); + result.setSuccessIsSet(true); + return result; + } + } + } @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class AsyncProcessor extends com.facebook.fb303.FacebookService.AsyncProcessor { @@ -19104,6 +19281,8 @@ protected AsyncProcessor(I iface, Map, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getMetaConf_args"); - - private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)1); - - private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); - static { - schemes.put(StandardScheme.class, new getMetaConf_argsStandardSchemeFactory()); - schemes.put(TupleScheme.class, new getMetaConf_argsTupleSchemeFactory()); - } - - private String key; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - KEY((short)1, "key"); - - private static final Map byName = new HashMap(); - - static { - for (_Fields field : EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 1: // KEY - return KEY; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - public static _Fields findByName(String name) { - return byName.get(name); - } - - private final short _thriftId; - private final String _fieldName; - - _Fields(short thriftId, String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - public short getThriftFieldId() { - return _thriftId; - } - - public String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); - metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getMetaConf_args.class, metaDataMap); - } - - public getMetaConf_args() { - } - - public getMetaConf_args( - String key) - { - this(); - this.key = key; - } - - /** - * Performs a deep copy on other. - */ - public getMetaConf_args(getMetaConf_args other) { - if (other.isSetKey()) { - this.key = other.key; - } - } - - public getMetaConf_args deepCopy() { - return new getMetaConf_args(this); - } - - @Override - public void clear() { - this.key = null; - } - - public String getKey() { - return this.key; - } - - public void setKey(String key) { - this.key = key; - } - - public void unsetKey() { - this.key = null; - } - - /** Returns true if field key is set (has been assigned a value) and false otherwise */ - public boolean isSetKey() { - return this.key != null; - } - - public void setKeyIsSet(boolean value) { - if (!value) { - this.key = null; - } - } - - public void setFieldValue(_Fields field, Object value) { - switch (field) { - case KEY: - if (value == null) { - unsetKey(); - } else { - setKey((String)value); - } - break; - - } - } - - public Object getFieldValue(_Fields field) { - switch (field) { - case KEY: - return getKey(); - + @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class get_lock_materialization_rebuild extends org.apache.thrift.AsyncProcessFunction { + public get_lock_materialization_rebuild() { + super("get_lock_materialization_rebuild"); } - throw new IllegalStateException(); - } - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - public boolean isSet(_Fields field) { - if (field == null) { - throw new IllegalArgumentException(); + public get_lock_materialization_rebuild_args getEmptyArgsInstance() { + return new get_lock_materialization_rebuild_args(); } - switch (field) { - case KEY: - return isSetKey(); + public AsyncMethodCallback getResultHandler(final AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new AsyncMethodCallback() { + public void onComplete(LockResponse o) { + get_lock_materialization_rebuild_result result = new get_lock_materialization_rebuild_result(); + result.success = o; + try { + fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); + return; + } catch (Exception e) { + LOGGER.error("Exception writing to internal frame buffer", e); + } + fb.close(); + } + public void onError(Exception e) { + byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; + org.apache.thrift.TBase msg; + get_lock_materialization_rebuild_result result = new get_lock_materialization_rebuild_result(); + { + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); + } + try { + fcall.sendResponse(fb,msg,msgType,seqid); + return; + } catch (Exception ex) { + LOGGER.error("Exception writing to internal frame buffer", ex); + } + fb.close(); + } + }; } - throw new IllegalStateException(); - } - @Override - public boolean equals(Object that) { - if (that == null) - return false; - if (that instanceof getMetaConf_args) - return this.equals((getMetaConf_args)that); - return false; - } - - public boolean equals(getMetaConf_args that) { - if (that == null) + protected boolean isOneway() { return false; - - boolean this_present_key = true && this.isSetKey(); - boolean that_present_key = true && that.isSetKey(); - if (this_present_key || that_present_key) { - if (!(this_present_key && that_present_key)) - return false; - if (!this.key.equals(that.key)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - List list = new ArrayList(); - - boolean present_key = true && (isSetKey()); - list.add(present_key); - if (present_key) - list.add(key); - - return list.hashCode(); - } - - @Override - public int compareTo(getMetaConf_args other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetKey()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - schemes.get(iprot.getScheme()).getScheme().read(iprot, this); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - schemes.get(oprot.getScheme()).getScheme().write(oprot, this); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("getMetaConf_args("); - boolean first = true; - - sb.append("key:"); - if (this.key == null) { - sb.append("null"); - } else { - sb.append(this.key); } - first = false; - sb.append(")"); - return sb.toString(); - } - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); + public void start(I iface, get_lock_materialization_rebuild_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws TException { + iface.get_lock_materialization_rebuild(args.dbName, args.tableName, args.txnId,resultHandler); } } - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); + @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class heartbeat_lock_materialization_rebuild extends org.apache.thrift.AsyncProcessFunction { + public heartbeat_lock_materialization_rebuild() { + super("heartbeat_lock_materialization_rebuild"); } - } - private static class getMetaConf_argsStandardSchemeFactory implements SchemeFactory { - public getMetaConf_argsStandardScheme getScheme() { - return new getMetaConf_argsStandardScheme(); + public heartbeat_lock_materialization_rebuild_args getEmptyArgsInstance() { + return new heartbeat_lock_materialization_rebuild_args(); } - } - - private static class getMetaConf_argsStandardScheme extends StandardScheme { - public void read(org.apache.thrift.protocol.TProtocol iprot, getMetaConf_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; + public AsyncMethodCallback getResultHandler(final AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new AsyncMethodCallback() { + public void onComplete(Boolean o) { + heartbeat_lock_materialization_rebuild_result result = new heartbeat_lock_materialization_rebuild_result(); + result.success = o; + result.setSuccessIsSet(true); + try { + fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); + return; + } catch (Exception e) { + LOGGER.error("Exception writing to internal frame buffer", e); + } + fb.close(); } - switch (schemeField.id) { - case 1: // KEY - if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.key = iprot.readString(); - struct.setKeyIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + public void onError(Exception e) { + byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; + org.apache.thrift.TBase msg; + heartbeat_lock_materialization_rebuild_result result = new heartbeat_lock_materialization_rebuild_result(); + { + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); + } + try { + fcall.sendResponse(fb,msg,msgType,seqid); + return; + } catch (Exception ex) { + LOGGER.error("Exception writing to internal frame buffer", ex); + } + fb.close(); } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - struct.validate(); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot, getMetaConf_args struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (struct.key != null) { - oprot.writeFieldBegin(KEY_FIELD_DESC); - oprot.writeString(struct.key); - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class getMetaConf_argsTupleSchemeFactory implements SchemeFactory { - public getMetaConf_argsTupleScheme getScheme() { - return new getMetaConf_argsTupleScheme(); + }; } - } - - private static class getMetaConf_argsTupleScheme extends TupleScheme { - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, getMetaConf_args struct) throws org.apache.thrift.TException { - TTupleProtocol oprot = (TTupleProtocol) prot; - BitSet optionals = new BitSet(); - if (struct.isSetKey()) { - optionals.set(0); - } - oprot.writeBitSet(optionals, 1); - if (struct.isSetKey()) { - oprot.writeString(struct.key); - } + protected boolean isOneway() { + return false; } - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, getMetaConf_args struct) throws org.apache.thrift.TException { - TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(1); - if (incoming.get(0)) { - struct.key = iprot.readString(); - struct.setKeyIsSet(true); - } + public void start(I iface, heartbeat_lock_materialization_rebuild_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws TException { + iface.heartbeat_lock_materialization_rebuild(args.dbName, args.tableName, args.txnId,resultHandler); } } } - @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class getMetaConf_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getMetaConf_result"); + @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class getMetaConf_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getMetaConf_args"); - private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0); - private static final org.apache.thrift.protocol.TField O1_FIELD_DESC = new org.apache.thrift.protocol.TField("o1", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField KEY_FIELD_DESC = new org.apache.thrift.protocol.TField("key", org.apache.thrift.protocol.TType.STRING, (short)1); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { - schemes.put(StandardScheme.class, new getMetaConf_resultStandardSchemeFactory()); - schemes.put(TupleScheme.class, new getMetaConf_resultTupleSchemeFactory()); + schemes.put(StandardScheme.class, new getMetaConf_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getMetaConf_argsTupleSchemeFactory()); } - private String success; // required - private MetaException o1; // required + private String key; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { - SUCCESS((short)0, "success"), - O1((short)1, "o1"); + KEY((short)1, "key"); private static final Map byName = new HashMap(); @@ -31782,10 +31703,371 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getMetaConf_args str */ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { - case 0: // SUCCESS - return SUCCESS; - case 1: // O1 - return O1; + case 1: // KEY + return KEY; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.KEY, new org.apache.thrift.meta_data.FieldMetaData("key", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getMetaConf_args.class, metaDataMap); + } + + public getMetaConf_args() { + } + + public getMetaConf_args( + String key) + { + this(); + this.key = key; + } + + /** + * Performs a deep copy on other. + */ + public getMetaConf_args(getMetaConf_args other) { + if (other.isSetKey()) { + this.key = other.key; + } + } + + public getMetaConf_args deepCopy() { + return new getMetaConf_args(this); + } + + @Override + public void clear() { + this.key = null; + } + + public String getKey() { + return this.key; + } + + public void setKey(String key) { + this.key = key; + } + + public void unsetKey() { + this.key = null; + } + + /** Returns true if field key is set (has been assigned a value) and false otherwise */ + public boolean isSetKey() { + return this.key != null; + } + + public void setKeyIsSet(boolean value) { + if (!value) { + this.key = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case KEY: + if (value == null) { + unsetKey(); + } else { + setKey((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case KEY: + return getKey(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case KEY: + return isSetKey(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof getMetaConf_args) + return this.equals((getMetaConf_args)that); + return false; + } + + public boolean equals(getMetaConf_args that) { + if (that == null) + return false; + + boolean this_present_key = true && this.isSetKey(); + boolean that_present_key = true && that.isSetKey(); + if (this_present_key || that_present_key) { + if (!(this_present_key && that_present_key)) + return false; + if (!this.key.equals(that.key)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_key = true && (isSetKey()); + list.add(present_key); + if (present_key) + list.add(key); + + return list.hashCode(); + } + + @Override + public int compareTo(getMetaConf_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetKey()).compareTo(other.isSetKey()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetKey()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.key, other.key); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("getMetaConf_args("); + boolean first = true; + + sb.append("key:"); + if (this.key == null) { + sb.append("null"); + } else { + sb.append(this.key); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getMetaConf_argsStandardSchemeFactory implements SchemeFactory { + public getMetaConf_argsStandardScheme getScheme() { + return new getMetaConf_argsStandardScheme(); + } + } + + private static class getMetaConf_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, getMetaConf_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // KEY + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.key = iprot.readString(); + struct.setKeyIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, getMetaConf_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.key != null) { + oprot.writeFieldBegin(KEY_FIELD_DESC); + oprot.writeString(struct.key); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getMetaConf_argsTupleSchemeFactory implements SchemeFactory { + public getMetaConf_argsTupleScheme getScheme() { + return new getMetaConf_argsTupleScheme(); + } + } + + private static class getMetaConf_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getMetaConf_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetKey()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetKey()) { + oprot.writeString(struct.key); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getMetaConf_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.key = iprot.readString(); + struct.setKeyIsSet(true); + } + } + } + + } + + @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class getMetaConf_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getMetaConf_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0); + private static final org.apache.thrift.protocol.TField O1_FIELD_DESC = new org.apache.thrift.protocol.TField("o1", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new getMetaConf_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new getMetaConf_resultTupleSchemeFactory()); + } + + private String success; // required + private MetaException o1; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + O1((short)1, "o1"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // O1 + return O1; default: return null; } @@ -231375,4 +231657,1853 @@ public void read(org.apache.thrift.protocol.TProtocol prot, get_serde_result str } + @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class get_lock_materialization_rebuild_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_lock_materialization_rebuild_args"); + + private static final org.apache.thrift.protocol.TField DB_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("dbName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField TXN_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("txnId", org.apache.thrift.protocol.TType.I64, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new get_lock_materialization_rebuild_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new get_lock_materialization_rebuild_argsTupleSchemeFactory()); + } + + private String dbName; // required + private String tableName; // required + private long txnId; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + DB_NAME((short)1, "dbName"), + TABLE_NAME((short)2, "tableName"), + TXN_ID((short)3, "txnId"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // DB_NAME + return DB_NAME; + case 2: // TABLE_NAME + return TABLE_NAME; + case 3: // TXN_ID + return TXN_ID; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TXNID_ISSET_ID = 0; + private byte __isset_bitfield = 0; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.DB_NAME, new org.apache.thrift.meta_data.FieldMetaData("dbName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.TXN_ID, new org.apache.thrift.meta_data.FieldMetaData("txnId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_lock_materialization_rebuild_args.class, metaDataMap); + } + + public get_lock_materialization_rebuild_args() { + } + + public get_lock_materialization_rebuild_args( + String dbName, + String tableName, + long txnId) + { + this(); + this.dbName = dbName; + this.tableName = tableName; + this.txnId = txnId; + setTxnIdIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public get_lock_materialization_rebuild_args(get_lock_materialization_rebuild_args other) { + __isset_bitfield = other.__isset_bitfield; + if (other.isSetDbName()) { + this.dbName = other.dbName; + } + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + this.txnId = other.txnId; + } + + public get_lock_materialization_rebuild_args deepCopy() { + return new get_lock_materialization_rebuild_args(this); + } + + @Override + public void clear() { + this.dbName = null; + this.tableName = null; + setTxnIdIsSet(false); + this.txnId = 0; + } + + public String getDbName() { + return this.dbName; + } + + public void setDbName(String dbName) { + this.dbName = dbName; + } + + public void unsetDbName() { + this.dbName = null; + } + + /** Returns true if field dbName is set (has been assigned a value) and false otherwise */ + public boolean isSetDbName() { + return this.dbName != null; + } + + public void setDbNameIsSet(boolean value) { + if (!value) { + this.dbName = null; + } + } + + public String getTableName() { + return this.tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + public long getTxnId() { + return this.txnId; + } + + public void setTxnId(long txnId) { + this.txnId = txnId; + setTxnIdIsSet(true); + } + + public void unsetTxnId() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __TXNID_ISSET_ID); + } + + /** Returns true if field txnId is set (has been assigned a value) and false otherwise */ + public boolean isSetTxnId() { + return EncodingUtils.testBit(__isset_bitfield, __TXNID_ISSET_ID); + } + + public void setTxnIdIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __TXNID_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case DB_NAME: + if (value == null) { + unsetDbName(); + } else { + setDbName((String)value); + } + break; + + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((String)value); + } + break; + + case TXN_ID: + if (value == null) { + unsetTxnId(); + } else { + setTxnId((Long)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case DB_NAME: + return getDbName(); + + case TABLE_NAME: + return getTableName(); + + case TXN_ID: + return getTxnId(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case DB_NAME: + return isSetDbName(); + case TABLE_NAME: + return isSetTableName(); + case TXN_ID: + return isSetTxnId(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof get_lock_materialization_rebuild_args) + return this.equals((get_lock_materialization_rebuild_args)that); + return false; + } + + public boolean equals(get_lock_materialization_rebuild_args that) { + if (that == null) + return false; + + boolean this_present_dbName = true && this.isSetDbName(); + boolean that_present_dbName = true && that.isSetDbName(); + if (this_present_dbName || that_present_dbName) { + if (!(this_present_dbName && that_present_dbName)) + return false; + if (!this.dbName.equals(that.dbName)) + return false; + } + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_txnId = true; + boolean that_present_txnId = true; + if (this_present_txnId || that_present_txnId) { + if (!(this_present_txnId && that_present_txnId)) + return false; + if (this.txnId != that.txnId) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_dbName = true && (isSetDbName()); + list.add(present_dbName); + if (present_dbName) + list.add(dbName); + + boolean present_tableName = true && (isSetTableName()); + list.add(present_tableName); + if (present_tableName) + list.add(tableName); + + boolean present_txnId = true; + list.add(present_txnId); + if (present_txnId) + list.add(txnId); + + return list.hashCode(); + } + + @Override + public int compareTo(get_lock_materialization_rebuild_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetDbName()).compareTo(other.isSetDbName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetDbName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.dbName, other.dbName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(other.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, other.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTxnId()).compareTo(other.isSetTxnId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTxnId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.txnId, other.txnId); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("get_lock_materialization_rebuild_args("); + boolean first = true; + + sb.append("dbName:"); + if (this.dbName == null) { + sb.append("null"); + } else { + sb.append(this.dbName); + } + first = false; + if (!first) sb.append(", "); + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("txnId:"); + sb.append(this.txnId); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class get_lock_materialization_rebuild_argsStandardSchemeFactory implements SchemeFactory { + public get_lock_materialization_rebuild_argsStandardScheme getScheme() { + return new get_lock_materialization_rebuild_argsStandardScheme(); + } + } + + private static class get_lock_materialization_rebuild_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, get_lock_materialization_rebuild_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // DB_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.dbName = iprot.readString(); + struct.setDbNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readString(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // TXN_ID + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.txnId = iprot.readI64(); + struct.setTxnIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, get_lock_materialization_rebuild_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.dbName != null) { + oprot.writeFieldBegin(DB_NAME_FIELD_DESC); + oprot.writeString(struct.dbName); + oprot.writeFieldEnd(); + } + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeString(struct.tableName); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TXN_ID_FIELD_DESC); + oprot.writeI64(struct.txnId); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class get_lock_materialization_rebuild_argsTupleSchemeFactory implements SchemeFactory { + public get_lock_materialization_rebuild_argsTupleScheme getScheme() { + return new get_lock_materialization_rebuild_argsTupleScheme(); + } + } + + private static class get_lock_materialization_rebuild_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, get_lock_materialization_rebuild_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetDbName()) { + optionals.set(0); + } + if (struct.isSetTableName()) { + optionals.set(1); + } + if (struct.isSetTxnId()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetDbName()) { + oprot.writeString(struct.dbName); + } + if (struct.isSetTableName()) { + oprot.writeString(struct.tableName); + } + if (struct.isSetTxnId()) { + oprot.writeI64(struct.txnId); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, get_lock_materialization_rebuild_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.dbName = iprot.readString(); + struct.setDbNameIsSet(true); + } + if (incoming.get(1)) { + struct.tableName = iprot.readString(); + struct.setTableNameIsSet(true); + } + if (incoming.get(2)) { + struct.txnId = iprot.readI64(); + struct.setTxnIdIsSet(true); + } + } + } + + } + + @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class get_lock_materialization_rebuild_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_lock_materialization_rebuild_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new get_lock_materialization_rebuild_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new get_lock_materialization_rebuild_resultTupleSchemeFactory()); + } + + private LockResponse success; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, LockResponse.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_lock_materialization_rebuild_result.class, metaDataMap); + } + + public get_lock_materialization_rebuild_result() { + } + + public get_lock_materialization_rebuild_result( + LockResponse success) + { + this(); + this.success = success; + } + + /** + * Performs a deep copy on other. + */ + public get_lock_materialization_rebuild_result(get_lock_materialization_rebuild_result other) { + if (other.isSetSuccess()) { + this.success = new LockResponse(other.success); + } + } + + public get_lock_materialization_rebuild_result deepCopy() { + return new get_lock_materialization_rebuild_result(this); + } + + @Override + public void clear() { + this.success = null; + } + + public LockResponse getSuccess() { + return this.success; + } + + public void setSuccess(LockResponse success) { + this.success = success; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((LockResponse)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof get_lock_materialization_rebuild_result) + return this.equals((get_lock_materialization_rebuild_result)that); + return false; + } + + public boolean equals(get_lock_materialization_rebuild_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_success = true && (isSetSuccess()); + list.add(present_success); + if (present_success) + list.add(success); + + return list.hashCode(); + } + + @Override + public int compareTo(get_lock_materialization_rebuild_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("get_lock_materialization_rebuild_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (success != null) { + success.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class get_lock_materialization_rebuild_resultStandardSchemeFactory implements SchemeFactory { + public get_lock_materialization_rebuild_resultStandardScheme getScheme() { + return new get_lock_materialization_rebuild_resultStandardScheme(); + } + } + + private static class get_lock_materialization_rebuild_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, get_lock_materialization_rebuild_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.success = new LockResponse(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, get_lock_materialization_rebuild_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + struct.success.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class get_lock_materialization_rebuild_resultTupleSchemeFactory implements SchemeFactory { + public get_lock_materialization_rebuild_resultTupleScheme getScheme() { + return new get_lock_materialization_rebuild_resultTupleScheme(); + } + } + + private static class get_lock_materialization_rebuild_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, get_lock_materialization_rebuild_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetSuccess()) { + struct.success.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, get_lock_materialization_rebuild_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.success = new LockResponse(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } + } + } + + } + + @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class heartbeat_lock_materialization_rebuild_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("heartbeat_lock_materialization_rebuild_args"); + + private static final org.apache.thrift.protocol.TField DB_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("dbName", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField TXN_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("txnId", org.apache.thrift.protocol.TType.I64, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new heartbeat_lock_materialization_rebuild_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new heartbeat_lock_materialization_rebuild_argsTupleSchemeFactory()); + } + + private String dbName; // required + private String tableName; // required + private long txnId; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + DB_NAME((short)1, "dbName"), + TABLE_NAME((short)2, "tableName"), + TXN_ID((short)3, "txnId"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // DB_NAME + return DB_NAME; + case 2: // TABLE_NAME + return TABLE_NAME; + case 3: // TXN_ID + return TXN_ID; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __TXNID_ISSET_ID = 0; + private byte __isset_bitfield = 0; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.DB_NAME, new org.apache.thrift.meta_data.FieldMetaData("dbName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.TABLE_NAME, new org.apache.thrift.meta_data.FieldMetaData("tableName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.TXN_ID, new org.apache.thrift.meta_data.FieldMetaData("txnId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(heartbeat_lock_materialization_rebuild_args.class, metaDataMap); + } + + public heartbeat_lock_materialization_rebuild_args() { + } + + public heartbeat_lock_materialization_rebuild_args( + String dbName, + String tableName, + long txnId) + { + this(); + this.dbName = dbName; + this.tableName = tableName; + this.txnId = txnId; + setTxnIdIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public heartbeat_lock_materialization_rebuild_args(heartbeat_lock_materialization_rebuild_args other) { + __isset_bitfield = other.__isset_bitfield; + if (other.isSetDbName()) { + this.dbName = other.dbName; + } + if (other.isSetTableName()) { + this.tableName = other.tableName; + } + this.txnId = other.txnId; + } + + public heartbeat_lock_materialization_rebuild_args deepCopy() { + return new heartbeat_lock_materialization_rebuild_args(this); + } + + @Override + public void clear() { + this.dbName = null; + this.tableName = null; + setTxnIdIsSet(false); + this.txnId = 0; + } + + public String getDbName() { + return this.dbName; + } + + public void setDbName(String dbName) { + this.dbName = dbName; + } + + public void unsetDbName() { + this.dbName = null; + } + + /** Returns true if field dbName is set (has been assigned a value) and false otherwise */ + public boolean isSetDbName() { + return this.dbName != null; + } + + public void setDbNameIsSet(boolean value) { + if (!value) { + this.dbName = null; + } + } + + public String getTableName() { + return this.tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public void unsetTableName() { + this.tableName = null; + } + + /** Returns true if field tableName is set (has been assigned a value) and false otherwise */ + public boolean isSetTableName() { + return this.tableName != null; + } + + public void setTableNameIsSet(boolean value) { + if (!value) { + this.tableName = null; + } + } + + public long getTxnId() { + return this.txnId; + } + + public void setTxnId(long txnId) { + this.txnId = txnId; + setTxnIdIsSet(true); + } + + public void unsetTxnId() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __TXNID_ISSET_ID); + } + + /** Returns true if field txnId is set (has been assigned a value) and false otherwise */ + public boolean isSetTxnId() { + return EncodingUtils.testBit(__isset_bitfield, __TXNID_ISSET_ID); + } + + public void setTxnIdIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __TXNID_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case DB_NAME: + if (value == null) { + unsetDbName(); + } else { + setDbName((String)value); + } + break; + + case TABLE_NAME: + if (value == null) { + unsetTableName(); + } else { + setTableName((String)value); + } + break; + + case TXN_ID: + if (value == null) { + unsetTxnId(); + } else { + setTxnId((Long)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case DB_NAME: + return getDbName(); + + case TABLE_NAME: + return getTableName(); + + case TXN_ID: + return getTxnId(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case DB_NAME: + return isSetDbName(); + case TABLE_NAME: + return isSetTableName(); + case TXN_ID: + return isSetTxnId(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof heartbeat_lock_materialization_rebuild_args) + return this.equals((heartbeat_lock_materialization_rebuild_args)that); + return false; + } + + public boolean equals(heartbeat_lock_materialization_rebuild_args that) { + if (that == null) + return false; + + boolean this_present_dbName = true && this.isSetDbName(); + boolean that_present_dbName = true && that.isSetDbName(); + if (this_present_dbName || that_present_dbName) { + if (!(this_present_dbName && that_present_dbName)) + return false; + if (!this.dbName.equals(that.dbName)) + return false; + } + + boolean this_present_tableName = true && this.isSetTableName(); + boolean that_present_tableName = true && that.isSetTableName(); + if (this_present_tableName || that_present_tableName) { + if (!(this_present_tableName && that_present_tableName)) + return false; + if (!this.tableName.equals(that.tableName)) + return false; + } + + boolean this_present_txnId = true; + boolean that_present_txnId = true; + if (this_present_txnId || that_present_txnId) { + if (!(this_present_txnId && that_present_txnId)) + return false; + if (this.txnId != that.txnId) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_dbName = true && (isSetDbName()); + list.add(present_dbName); + if (present_dbName) + list.add(dbName); + + boolean present_tableName = true && (isSetTableName()); + list.add(present_tableName); + if (present_tableName) + list.add(tableName); + + boolean present_txnId = true; + list.add(present_txnId); + if (present_txnId) + list.add(txnId); + + return list.hashCode(); + } + + @Override + public int compareTo(heartbeat_lock_materialization_rebuild_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetDbName()).compareTo(other.isSetDbName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetDbName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.dbName, other.dbName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTableName()).compareTo(other.isSetTableName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTableName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tableName, other.tableName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetTxnId()).compareTo(other.isSetTxnId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTxnId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.txnId, other.txnId); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("heartbeat_lock_materialization_rebuild_args("); + boolean first = true; + + sb.append("dbName:"); + if (this.dbName == null) { + sb.append("null"); + } else { + sb.append(this.dbName); + } + first = false; + if (!first) sb.append(", "); + sb.append("tableName:"); + if (this.tableName == null) { + sb.append("null"); + } else { + sb.append(this.tableName); + } + first = false; + if (!first) sb.append(", "); + sb.append("txnId:"); + sb.append(this.txnId); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class heartbeat_lock_materialization_rebuild_argsStandardSchemeFactory implements SchemeFactory { + public heartbeat_lock_materialization_rebuild_argsStandardScheme getScheme() { + return new heartbeat_lock_materialization_rebuild_argsStandardScheme(); + } + } + + private static class heartbeat_lock_materialization_rebuild_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, heartbeat_lock_materialization_rebuild_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // DB_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.dbName = iprot.readString(); + struct.setDbNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // TABLE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tableName = iprot.readString(); + struct.setTableNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // TXN_ID + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.txnId = iprot.readI64(); + struct.setTxnIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, heartbeat_lock_materialization_rebuild_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.dbName != null) { + oprot.writeFieldBegin(DB_NAME_FIELD_DESC); + oprot.writeString(struct.dbName); + oprot.writeFieldEnd(); + } + if (struct.tableName != null) { + oprot.writeFieldBegin(TABLE_NAME_FIELD_DESC); + oprot.writeString(struct.tableName); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TXN_ID_FIELD_DESC); + oprot.writeI64(struct.txnId); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class heartbeat_lock_materialization_rebuild_argsTupleSchemeFactory implements SchemeFactory { + public heartbeat_lock_materialization_rebuild_argsTupleScheme getScheme() { + return new heartbeat_lock_materialization_rebuild_argsTupleScheme(); + } + } + + private static class heartbeat_lock_materialization_rebuild_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, heartbeat_lock_materialization_rebuild_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetDbName()) { + optionals.set(0); + } + if (struct.isSetTableName()) { + optionals.set(1); + } + if (struct.isSetTxnId()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetDbName()) { + oprot.writeString(struct.dbName); + } + if (struct.isSetTableName()) { + oprot.writeString(struct.tableName); + } + if (struct.isSetTxnId()) { + oprot.writeI64(struct.txnId); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, heartbeat_lock_materialization_rebuild_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.dbName = iprot.readString(); + struct.setDbNameIsSet(true); + } + if (incoming.get(1)) { + struct.tableName = iprot.readString(); + struct.setTableNameIsSet(true); + } + if (incoming.get(2)) { + struct.txnId = iprot.readI64(); + struct.setTxnIdIsSet(true); + } + } + } + + } + + @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class heartbeat_lock_materialization_rebuild_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("heartbeat_lock_materialization_rebuild_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.BOOL, (short)0); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new heartbeat_lock_materialization_rebuild_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new heartbeat_lock_materialization_rebuild_resultTupleSchemeFactory()); + } + + private boolean success; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private byte __isset_bitfield = 0; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(heartbeat_lock_materialization_rebuild_result.class, metaDataMap); + } + + public heartbeat_lock_materialization_rebuild_result() { + } + + public heartbeat_lock_materialization_rebuild_result( + boolean success) + { + this(); + this.success = success; + setSuccessIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public heartbeat_lock_materialization_rebuild_result(heartbeat_lock_materialization_rebuild_result other) { + __isset_bitfield = other.__isset_bitfield; + this.success = other.success; + } + + public heartbeat_lock_materialization_rebuild_result deepCopy() { + return new heartbeat_lock_materialization_rebuild_result(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = false; + } + + public boolean isSuccess() { + return this.success; + } + + public void setSuccess(boolean success) { + this.success = success; + setSuccessIsSet(true); + } + + public void unsetSuccess() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return EncodingUtils.testBit(__isset_bitfield, __SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __SUCCESS_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Boolean)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return isSuccess(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof heartbeat_lock_materialization_rebuild_result) + return this.equals((heartbeat_lock_materialization_rebuild_result)that); + return false; + } + + public boolean equals(heartbeat_lock_materialization_rebuild_result that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_success = true; + list.add(present_success); + if (present_success) + list.add(success); + + return list.hashCode(); + } + + @Override + public int compareTo(heartbeat_lock_materialization_rebuild_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("heartbeat_lock_materialization_rebuild_result("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class heartbeat_lock_materialization_rebuild_resultStandardSchemeFactory implements SchemeFactory { + public heartbeat_lock_materialization_rebuild_resultStandardScheme getScheme() { + return new heartbeat_lock_materialization_rebuild_resultStandardScheme(); + } + } + + private static class heartbeat_lock_materialization_rebuild_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, heartbeat_lock_materialization_rebuild_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, heartbeat_lock_materialization_rebuild_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.isSetSuccess()) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeBool(struct.success); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class heartbeat_lock_materialization_rebuild_resultTupleSchemeFactory implements SchemeFactory { + public heartbeat_lock_materialization_rebuild_resultTupleScheme getScheme() { + return new heartbeat_lock_materialization_rebuild_resultTupleScheme(); + } + } + + private static class heartbeat_lock_materialization_rebuild_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, heartbeat_lock_materialization_rebuild_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetSuccess()) { + oprot.writeBool(struct.success); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, heartbeat_lock_materialization_rebuild_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } + } + } + + } + } diff --git a/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php b/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php index 5e3dff1a12..7a8a42a800 100644 --- a/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php +++ b/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php @@ -1514,6 +1514,20 @@ interface ThriftHiveMetastoreIf extends \FacebookServiceIf { * @throws \metastore\MetaException */ public function get_serde(\metastore\GetSerdeRequest $rqst); + /** + * @param string $dbName + * @param string $tableName + * @param int $txnId + * @return \metastore\LockResponse + */ + public function get_lock_materialization_rebuild($dbName, $tableName, $txnId); + /** + * @param string $dbName + * @param string $tableName + * @param int $txnId + * @return bool + */ + public function heartbeat_lock_materialization_rebuild($dbName, $tableName, $txnId); } class ThriftHiveMetastoreClient extends \FacebookServiceClient implements \metastore\ThriftHiveMetastoreIf { @@ -12887,6 +12901,112 @@ class ThriftHiveMetastoreClient extends \FacebookServiceClient implements \metas throw new \Exception("get_serde failed: unknown result"); } + public function get_lock_materialization_rebuild($dbName, $tableName, $txnId) + { + $this->send_get_lock_materialization_rebuild($dbName, $tableName, $txnId); + return $this->recv_get_lock_materialization_rebuild(); + } + + public function send_get_lock_materialization_rebuild($dbName, $tableName, $txnId) + { + $args = new \metastore\ThriftHiveMetastore_get_lock_materialization_rebuild_args(); + $args->dbName = $dbName; + $args->tableName = $tableName; + $args->txnId = $txnId; + $bin_accel = ($this->output_ instanceof TBinaryProtocolAccelerated) && function_exists('thrift_protocol_write_binary'); + if ($bin_accel) + { + thrift_protocol_write_binary($this->output_, 'get_lock_materialization_rebuild', TMessageType::CALL, $args, $this->seqid_, $this->output_->isStrictWrite()); + } + else + { + $this->output_->writeMessageBegin('get_lock_materialization_rebuild', TMessageType::CALL, $this->seqid_); + $args->write($this->output_); + $this->output_->writeMessageEnd(); + $this->output_->getTransport()->flush(); + } + } + + public function recv_get_lock_materialization_rebuild() + { + $bin_accel = ($this->input_ instanceof TBinaryProtocolAccelerated) && function_exists('thrift_protocol_read_binary'); + if ($bin_accel) $result = thrift_protocol_read_binary($this->input_, '\metastore\ThriftHiveMetastore_get_lock_materialization_rebuild_result', $this->input_->isStrictRead()); + else + { + $rseqid = 0; + $fname = null; + $mtype = 0; + + $this->input_->readMessageBegin($fname, $mtype, $rseqid); + if ($mtype == TMessageType::EXCEPTION) { + $x = new TApplicationException(); + $x->read($this->input_); + $this->input_->readMessageEnd(); + throw $x; + } + $result = new \metastore\ThriftHiveMetastore_get_lock_materialization_rebuild_result(); + $result->read($this->input_); + $this->input_->readMessageEnd(); + } + if ($result->success !== null) { + return $result->success; + } + throw new \Exception("get_lock_materialization_rebuild failed: unknown result"); + } + + public function heartbeat_lock_materialization_rebuild($dbName, $tableName, $txnId) + { + $this->send_heartbeat_lock_materialization_rebuild($dbName, $tableName, $txnId); + return $this->recv_heartbeat_lock_materialization_rebuild(); + } + + public function send_heartbeat_lock_materialization_rebuild($dbName, $tableName, $txnId) + { + $args = new \metastore\ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args(); + $args->dbName = $dbName; + $args->tableName = $tableName; + $args->txnId = $txnId; + $bin_accel = ($this->output_ instanceof TBinaryProtocolAccelerated) && function_exists('thrift_protocol_write_binary'); + if ($bin_accel) + { + thrift_protocol_write_binary($this->output_, 'heartbeat_lock_materialization_rebuild', TMessageType::CALL, $args, $this->seqid_, $this->output_->isStrictWrite()); + } + else + { + $this->output_->writeMessageBegin('heartbeat_lock_materialization_rebuild', TMessageType::CALL, $this->seqid_); + $args->write($this->output_); + $this->output_->writeMessageEnd(); + $this->output_->getTransport()->flush(); + } + } + + public function recv_heartbeat_lock_materialization_rebuild() + { + $bin_accel = ($this->input_ instanceof TBinaryProtocolAccelerated) && function_exists('thrift_protocol_read_binary'); + if ($bin_accel) $result = thrift_protocol_read_binary($this->input_, '\metastore\ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result', $this->input_->isStrictRead()); + else + { + $rseqid = 0; + $fname = null; + $mtype = 0; + + $this->input_->readMessageBegin($fname, $mtype, $rseqid); + if ($mtype == TMessageType::EXCEPTION) { + $x = new TApplicationException(); + $x->read($this->input_); + $this->input_->readMessageEnd(); + throw $x; + } + $result = new \metastore\ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result(); + $result->read($this->input_); + $this->input_->readMessageEnd(); + } + if ($result->success !== null) { + return $result->success; + } + throw new \Exception("heartbeat_lock_materialization_rebuild failed: unknown result"); + } + } // HELPER FUNCTIONS AND STRUCTURES @@ -57981,4 +58101,401 @@ class ThriftHiveMetastore_get_serde_result { } +class ThriftHiveMetastore_get_lock_materialization_rebuild_args { + static $_TSPEC; + + /** + * @var string + */ + public $dbName = null; + /** + * @var string + */ + public $tableName = null; + /** + * @var int + */ + public $txnId = null; + + public function __construct($vals=null) { + if (!isset(self::$_TSPEC)) { + self::$_TSPEC = array( + 1 => array( + 'var' => 'dbName', + 'type' => TType::STRING, + ), + 2 => array( + 'var' => 'tableName', + 'type' => TType::STRING, + ), + 3 => array( + 'var' => 'txnId', + 'type' => TType::I64, + ), + ); + } + if (is_array($vals)) { + if (isset($vals['dbName'])) { + $this->dbName = $vals['dbName']; + } + if (isset($vals['tableName'])) { + $this->tableName = $vals['tableName']; + } + if (isset($vals['txnId'])) { + $this->txnId = $vals['txnId']; + } + } + } + + public function getName() { + return 'ThriftHiveMetastore_get_lock_materialization_rebuild_args'; + } + + public function read($input) + { + $xfer = 0; + $fname = null; + $ftype = 0; + $fid = 0; + $xfer += $input->readStructBegin($fname); + while (true) + { + $xfer += $input->readFieldBegin($fname, $ftype, $fid); + if ($ftype == TType::STOP) { + break; + } + switch ($fid) + { + case 1: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->dbName); + } else { + $xfer += $input->skip($ftype); + } + break; + case 2: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->tableName); + } else { + $xfer += $input->skip($ftype); + } + break; + case 3: + if ($ftype == TType::I64) { + $xfer += $input->readI64($this->txnId); + } else { + $xfer += $input->skip($ftype); + } + break; + default: + $xfer += $input->skip($ftype); + break; + } + $xfer += $input->readFieldEnd(); + } + $xfer += $input->readStructEnd(); + return $xfer; + } + + public function write($output) { + $xfer = 0; + $xfer += $output->writeStructBegin('ThriftHiveMetastore_get_lock_materialization_rebuild_args'); + if ($this->dbName !== null) { + $xfer += $output->writeFieldBegin('dbName', TType::STRING, 1); + $xfer += $output->writeString($this->dbName); + $xfer += $output->writeFieldEnd(); + } + if ($this->tableName !== null) { + $xfer += $output->writeFieldBegin('tableName', TType::STRING, 2); + $xfer += $output->writeString($this->tableName); + $xfer += $output->writeFieldEnd(); + } + if ($this->txnId !== null) { + $xfer += $output->writeFieldBegin('txnId', TType::I64, 3); + $xfer += $output->writeI64($this->txnId); + $xfer += $output->writeFieldEnd(); + } + $xfer += $output->writeFieldStop(); + $xfer += $output->writeStructEnd(); + return $xfer; + } + +} + +class ThriftHiveMetastore_get_lock_materialization_rebuild_result { + static $_TSPEC; + + /** + * @var \metastore\LockResponse + */ + public $success = null; + + public function __construct($vals=null) { + if (!isset(self::$_TSPEC)) { + self::$_TSPEC = array( + 0 => array( + 'var' => 'success', + 'type' => TType::STRUCT, + 'class' => '\metastore\LockResponse', + ), + ); + } + if (is_array($vals)) { + if (isset($vals['success'])) { + $this->success = $vals['success']; + } + } + } + + public function getName() { + return 'ThriftHiveMetastore_get_lock_materialization_rebuild_result'; + } + + public function read($input) + { + $xfer = 0; + $fname = null; + $ftype = 0; + $fid = 0; + $xfer += $input->readStructBegin($fname); + while (true) + { + $xfer += $input->readFieldBegin($fname, $ftype, $fid); + if ($ftype == TType::STOP) { + break; + } + switch ($fid) + { + case 0: + if ($ftype == TType::STRUCT) { + $this->success = new \metastore\LockResponse(); + $xfer += $this->success->read($input); + } else { + $xfer += $input->skip($ftype); + } + break; + default: + $xfer += $input->skip($ftype); + break; + } + $xfer += $input->readFieldEnd(); + } + $xfer += $input->readStructEnd(); + return $xfer; + } + + public function write($output) { + $xfer = 0; + $xfer += $output->writeStructBegin('ThriftHiveMetastore_get_lock_materialization_rebuild_result'); + if ($this->success !== null) { + if (!is_object($this->success)) { + throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA); + } + $xfer += $output->writeFieldBegin('success', TType::STRUCT, 0); + $xfer += $this->success->write($output); + $xfer += $output->writeFieldEnd(); + } + $xfer += $output->writeFieldStop(); + $xfer += $output->writeStructEnd(); + return $xfer; + } + +} + +class ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args { + static $_TSPEC; + + /** + * @var string + */ + public $dbName = null; + /** + * @var string + */ + public $tableName = null; + /** + * @var int + */ + public $txnId = null; + + public function __construct($vals=null) { + if (!isset(self::$_TSPEC)) { + self::$_TSPEC = array( + 1 => array( + 'var' => 'dbName', + 'type' => TType::STRING, + ), + 2 => array( + 'var' => 'tableName', + 'type' => TType::STRING, + ), + 3 => array( + 'var' => 'txnId', + 'type' => TType::I64, + ), + ); + } + if (is_array($vals)) { + if (isset($vals['dbName'])) { + $this->dbName = $vals['dbName']; + } + if (isset($vals['tableName'])) { + $this->tableName = $vals['tableName']; + } + if (isset($vals['txnId'])) { + $this->txnId = $vals['txnId']; + } + } + } + + public function getName() { + return 'ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args'; + } + + public function read($input) + { + $xfer = 0; + $fname = null; + $ftype = 0; + $fid = 0; + $xfer += $input->readStructBegin($fname); + while (true) + { + $xfer += $input->readFieldBegin($fname, $ftype, $fid); + if ($ftype == TType::STOP) { + break; + } + switch ($fid) + { + case 1: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->dbName); + } else { + $xfer += $input->skip($ftype); + } + break; + case 2: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->tableName); + } else { + $xfer += $input->skip($ftype); + } + break; + case 3: + if ($ftype == TType::I64) { + $xfer += $input->readI64($this->txnId); + } else { + $xfer += $input->skip($ftype); + } + break; + default: + $xfer += $input->skip($ftype); + break; + } + $xfer += $input->readFieldEnd(); + } + $xfer += $input->readStructEnd(); + return $xfer; + } + + public function write($output) { + $xfer = 0; + $xfer += $output->writeStructBegin('ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_args'); + if ($this->dbName !== null) { + $xfer += $output->writeFieldBegin('dbName', TType::STRING, 1); + $xfer += $output->writeString($this->dbName); + $xfer += $output->writeFieldEnd(); + } + if ($this->tableName !== null) { + $xfer += $output->writeFieldBegin('tableName', TType::STRING, 2); + $xfer += $output->writeString($this->tableName); + $xfer += $output->writeFieldEnd(); + } + if ($this->txnId !== null) { + $xfer += $output->writeFieldBegin('txnId', TType::I64, 3); + $xfer += $output->writeI64($this->txnId); + $xfer += $output->writeFieldEnd(); + } + $xfer += $output->writeFieldStop(); + $xfer += $output->writeStructEnd(); + return $xfer; + } + +} + +class ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result { + static $_TSPEC; + + /** + * @var bool + */ + public $success = null; + + public function __construct($vals=null) { + if (!isset(self::$_TSPEC)) { + self::$_TSPEC = array( + 0 => array( + 'var' => 'success', + 'type' => TType::BOOL, + ), + ); + } + if (is_array($vals)) { + if (isset($vals['success'])) { + $this->success = $vals['success']; + } + } + } + + public function getName() { + return 'ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result'; + } + + public function read($input) + { + $xfer = 0; + $fname = null; + $ftype = 0; + $fid = 0; + $xfer += $input->readStructBegin($fname); + while (true) + { + $xfer += $input->readFieldBegin($fname, $ftype, $fid); + if ($ftype == TType::STOP) { + break; + } + switch ($fid) + { + case 0: + if ($ftype == TType::BOOL) { + $xfer += $input->readBool($this->success); + } else { + $xfer += $input->skip($ftype); + } + break; + default: + $xfer += $input->skip($ftype); + break; + } + $xfer += $input->readFieldEnd(); + } + $xfer += $input->readStructEnd(); + return $xfer; + } + + public function write($output) { + $xfer = 0; + $xfer += $output->writeStructBegin('ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_result'); + if ($this->success !== null) { + $xfer += $output->writeFieldBegin('success', TType::BOOL, 0); + $xfer += $output->writeBool($this->success); + $xfer += $output->writeFieldEnd(); + } + $xfer += $output->writeFieldStop(); + $xfer += $output->writeStructEnd(); + return $xfer; + } + +} + diff --git a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php index d4fcc888ed..14416b4875 100644 --- a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php +++ b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php @@ -23975,6 +23975,10 @@ class Materialization { * @var int */ public $invalidationTime = null; + /** + * @var bool + */ + public $sourceTablesUpdateDeleteModified = null; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -23995,6 +23999,10 @@ class Materialization { 'var' => 'invalidationTime', 'type' => TType::I64, ), + 4 => array( + 'var' => 'sourceTablesUpdateDeleteModified', + 'type' => TType::BOOL, + ), ); } if (is_array($vals)) { @@ -24007,6 +24015,9 @@ class Materialization { if (isset($vals['invalidationTime'])) { $this->invalidationTime = $vals['invalidationTime']; } + if (isset($vals['sourceTablesUpdateDeleteModified'])) { + $this->sourceTablesUpdateDeleteModified = $vals['sourceTablesUpdateDeleteModified']; + } } } @@ -24064,6 +24075,13 @@ class Materialization { $xfer += $input->skip($ftype); } break; + case 4: + if ($ftype == TType::BOOL) { + $xfer += $input->readBool($this->sourceTablesUpdateDeleteModified); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -24108,6 +24126,11 @@ class Materialization { $xfer += $output->writeI64($this->invalidationTime); $xfer += $output->writeFieldEnd(); } + if ($this->sourceTablesUpdateDeleteModified !== null) { + $xfer += $output->writeFieldBegin('sourceTablesUpdateDeleteModified', TType::BOOL, 4); + $xfer += $output->writeBool($this->sourceTablesUpdateDeleteModified); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; diff --git a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote index d39690f31c..079c7fc322 100755 --- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote +++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote @@ -224,6 +224,8 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print(' void set_schema_version_state(SetSchemaVersionStateRequest rqst)') print(' void add_serde(SerDeInfo serde)') print(' SerDeInfo get_serde(GetSerdeRequest rqst)') + print(' LockResponse get_lock_materialization_rebuild(string dbName, string tableName, i64 txnId)') + print(' bool heartbeat_lock_materialization_rebuild(string dbName, string tableName, i64 txnId)') print(' string getName()') print(' string getVersion()') print(' fb_status getStatus()') @@ -1493,6 +1495,18 @@ elif cmd == 'get_serde': sys.exit(1) pp.pprint(client.get_serde(eval(args[0]),)) +elif cmd == 'get_lock_materialization_rebuild': + if len(args) != 3: + print('get_lock_materialization_rebuild requires 3 args') + sys.exit(1) + pp.pprint(client.get_lock_materialization_rebuild(args[0],args[1],eval(args[2]),)) + +elif cmd == 'heartbeat_lock_materialization_rebuild': + if len(args) != 3: + print('heartbeat_lock_materialization_rebuild requires 3 args') + sys.exit(1) + pp.pprint(client.heartbeat_lock_materialization_rebuild(args[0],args[1],eval(args[2]),)) + elif cmd == 'getName': if len(args) != 0: print('getName requires 0 args') diff --git a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py index f8ffeac605..b0e64d8cef 100644 --- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py +++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py @@ -1548,6 +1548,24 @@ def get_serde(self, rqst): """ pass + def get_lock_materialization_rebuild(self, dbName, tableName, txnId): + """ + Parameters: + - dbName + - tableName + - txnId + """ + pass + + def heartbeat_lock_materialization_rebuild(self, dbName, tableName, txnId): + """ + Parameters: + - dbName + - tableName + - txnId + """ + pass + class Client(fb303.FacebookService.Client, Iface): """ @@ -8711,6 +8729,76 @@ def recv_get_serde(self): raise result.o2 raise TApplicationException(TApplicationException.MISSING_RESULT, "get_serde failed: unknown result") + def get_lock_materialization_rebuild(self, dbName, tableName, txnId): + """ + Parameters: + - dbName + - tableName + - txnId + """ + self.send_get_lock_materialization_rebuild(dbName, tableName, txnId) + return self.recv_get_lock_materialization_rebuild() + + def send_get_lock_materialization_rebuild(self, dbName, tableName, txnId): + self._oprot.writeMessageBegin('get_lock_materialization_rebuild', TMessageType.CALL, self._seqid) + args = get_lock_materialization_rebuild_args() + args.dbName = dbName + args.tableName = tableName + args.txnId = txnId + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_lock_materialization_rebuild(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_lock_materialization_rebuild_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_lock_materialization_rebuild failed: unknown result") + + def heartbeat_lock_materialization_rebuild(self, dbName, tableName, txnId): + """ + Parameters: + - dbName + - tableName + - txnId + """ + self.send_heartbeat_lock_materialization_rebuild(dbName, tableName, txnId) + return self.recv_heartbeat_lock_materialization_rebuild() + + def send_heartbeat_lock_materialization_rebuild(self, dbName, tableName, txnId): + self._oprot.writeMessageBegin('heartbeat_lock_materialization_rebuild', TMessageType.CALL, self._seqid) + args = heartbeat_lock_materialization_rebuild_args() + args.dbName = dbName + args.tableName = tableName + args.txnId = txnId + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_heartbeat_lock_materialization_rebuild(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = heartbeat_lock_materialization_rebuild_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "heartbeat_lock_materialization_rebuild failed: unknown result") + class Processor(fb303.FacebookService.Processor, Iface, TProcessor): def __init__(self, handler): @@ -8915,6 +9003,8 @@ def __init__(self, handler): self._processMap["set_schema_version_state"] = Processor.process_set_schema_version_state self._processMap["add_serde"] = Processor.process_add_serde self._processMap["get_serde"] = Processor.process_get_serde + self._processMap["get_lock_materialization_rebuild"] = Processor.process_get_lock_materialization_rebuild + self._processMap["heartbeat_lock_materialization_rebuild"] = Processor.process_heartbeat_lock_materialization_rebuild def process(self, iprot, oprot): (name, type, seqid) = iprot.readMessageBegin() @@ -13889,6 +13979,44 @@ def process_get_serde(self, seqid, iprot, oprot): oprot.writeMessageEnd() oprot.trans.flush() + def process_get_lock_materialization_rebuild(self, seqid, iprot, oprot): + args = get_lock_materialization_rebuild_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_lock_materialization_rebuild_result() + try: + result.success = self._handler.get_lock_materialization_rebuild(args.dbName, args.tableName, args.txnId) + msg_type = TMessageType.REPLY + except (TTransport.TTransportException, KeyboardInterrupt, SystemExit): + raise + except Exception as ex: + msg_type = TMessageType.EXCEPTION + logging.exception(ex) + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("get_lock_materialization_rebuild", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_heartbeat_lock_materialization_rebuild(self, seqid, iprot, oprot): + args = heartbeat_lock_materialization_rebuild_args() + args.read(iprot) + iprot.readMessageEnd() + result = heartbeat_lock_materialization_rebuild_result() + try: + result.success = self._handler.heartbeat_lock_materialization_rebuild(args.dbName, args.tableName, args.txnId) + msg_type = TMessageType.REPLY + except (TTransport.TTransportException, KeyboardInterrupt, SystemExit): + raise + except Exception as ex: + msg_type = TMessageType.EXCEPTION + logging.exception(ex) + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("heartbeat_lock_materialization_rebuild", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + # HELPER FUNCTIONS AND STRUCTURES @@ -47281,3 +47409,314 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + +class get_lock_materialization_rebuild_args: + """ + Attributes: + - dbName + - tableName + - txnId + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'dbName', None, None, ), # 1 + (2, TType.STRING, 'tableName', None, None, ), # 2 + (3, TType.I64, 'txnId', None, None, ), # 3 + ) + + def __init__(self, dbName=None, tableName=None, txnId=None,): + self.dbName = dbName + self.tableName = tableName + self.txnId = txnId + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tableName = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.txnId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('get_lock_materialization_rebuild_args') + if self.dbName is not None: + oprot.writeFieldBegin('dbName', TType.STRING, 1) + oprot.writeString(self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin('tableName', TType.STRING, 2) + oprot.writeString(self.tableName) + oprot.writeFieldEnd() + if self.txnId is not None: + oprot.writeFieldBegin('txnId', TType.I64, 3) + oprot.writeI64(self.txnId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __hash__(self): + value = 17 + value = (value * 31) ^ hash(self.dbName) + value = (value * 31) ^ hash(self.tableName) + value = (value * 31) ^ hash(self.txnId) + return value + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class get_lock_materialization_rebuild_result: + """ + Attributes: + - success + """ + + thrift_spec = ( + (0, TType.STRUCT, 'success', (LockResponse, LockResponse.thrift_spec), None, ), # 0 + ) + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = LockResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('get_lock_materialization_rebuild_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __hash__(self): + value = 17 + value = (value * 31) ^ hash(self.success) + return value + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class heartbeat_lock_materialization_rebuild_args: + """ + Attributes: + - dbName + - tableName + - txnId + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'dbName', None, None, ), # 1 + (2, TType.STRING, 'tableName', None, None, ), # 2 + (3, TType.I64, 'txnId', None, None, ), # 3 + ) + + def __init__(self, dbName=None, tableName=None, txnId=None,): + self.dbName = dbName + self.tableName = tableName + self.txnId = txnId + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tableName = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.txnId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('heartbeat_lock_materialization_rebuild_args') + if self.dbName is not None: + oprot.writeFieldBegin('dbName', TType.STRING, 1) + oprot.writeString(self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin('tableName', TType.STRING, 2) + oprot.writeString(self.tableName) + oprot.writeFieldEnd() + if self.txnId is not None: + oprot.writeFieldBegin('txnId', TType.I64, 3) + oprot.writeI64(self.txnId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __hash__(self): + value = 17 + value = (value * 31) ^ hash(self.dbName) + value = (value * 31) ^ hash(self.tableName) + value = (value * 31) ^ hash(self.txnId) + return value + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class heartbeat_lock_materialization_rebuild_result: + """ + Attributes: + - success + """ + + thrift_spec = ( + (0, TType.BOOL, 'success', None, None, ), # 0 + ) + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('heartbeat_lock_materialization_rebuild_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __hash__(self): + value = 17 + value = (value * 31) ^ hash(self.success) + return value + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) diff --git a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py index 972db1f0a3..f2f61e0499 100644 --- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -16941,6 +16941,7 @@ class Materialization: - tablesUsed - validTxnList - invalidationTime + - sourceTablesUpdateDeleteModified """ thrift_spec = ( @@ -16948,12 +16949,14 @@ class Materialization: (1, TType.SET, 'tablesUsed', (TType.STRING,None), None, ), # 1 (2, TType.STRING, 'validTxnList', None, None, ), # 2 (3, TType.I64, 'invalidationTime', None, None, ), # 3 + (4, TType.BOOL, 'sourceTablesUpdateDeleteModified', None, None, ), # 4 ) - def __init__(self, tablesUsed=None, validTxnList=None, invalidationTime=None,): + def __init__(self, tablesUsed=None, validTxnList=None, invalidationTime=None, sourceTablesUpdateDeleteModified=None,): self.tablesUsed = tablesUsed self.validTxnList = validTxnList self.invalidationTime = invalidationTime + self.sourceTablesUpdateDeleteModified = sourceTablesUpdateDeleteModified def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -16984,6 +16987,11 @@ def read(self, iprot): self.invalidationTime = iprot.readI64() else: iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.sourceTablesUpdateDeleteModified = iprot.readBool() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -17009,14 +17017,16 @@ def write(self, oprot): oprot.writeFieldBegin('invalidationTime', TType.I64, 3) oprot.writeI64(self.invalidationTime) oprot.writeFieldEnd() + if self.sourceTablesUpdateDeleteModified is not None: + oprot.writeFieldBegin('sourceTablesUpdateDeleteModified', TType.BOOL, 4) + oprot.writeBool(self.sourceTablesUpdateDeleteModified) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): if self.tablesUsed is None: raise TProtocol.TProtocolException(message='Required field tablesUsed is unset!') - if self.invalidationTime is None: - raise TProtocol.TProtocolException(message='Required field invalidationTime is unset!') return @@ -17025,6 +17035,7 @@ def __hash__(self): value = (value * 31) ^ hash(self.tablesUsed) value = (value * 31) ^ hash(self.validTxnList) value = (value * 31) ^ hash(self.invalidationTime) + value = (value * 31) ^ hash(self.sourceTablesUpdateDeleteModified) return value def __repr__(self): diff --git a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb index 94454a1499..0e70e8900d 100644 --- a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -3814,18 +3814,19 @@ class Materialization TABLESUSED = 1 VALIDTXNLIST = 2 INVALIDATIONTIME = 3 + SOURCETABLESUPDATEDELETEMODIFIED = 4 FIELDS = { TABLESUSED => {:type => ::Thrift::Types::SET, :name => 'tablesUsed', :element => {:type => ::Thrift::Types::STRING}}, VALIDTXNLIST => {:type => ::Thrift::Types::STRING, :name => 'validTxnList', :optional => true}, - INVALIDATIONTIME => {:type => ::Thrift::Types::I64, :name => 'invalidationTime'} + INVALIDATIONTIME => {:type => ::Thrift::Types::I64, :name => 'invalidationTime', :optional => true}, + SOURCETABLESUPDATEDELETEMODIFIED => {:type => ::Thrift::Types::BOOL, :name => 'sourceTablesUpdateDeleteModified', :optional => true} } def struct_fields; FIELDS; end def validate raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tablesUsed is unset!') unless @tablesUsed - raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field invalidationTime is unset!') unless @invalidationTime end ::Thrift::Struct.generate_accessors self diff --git a/standalone-metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb b/standalone-metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb index c1036755b4..58ebd29523 100644 --- a/standalone-metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb +++ b/standalone-metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb @@ -3348,6 +3348,36 @@ module ThriftHiveMetastore raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_serde failed: unknown result') end + def get_lock_materialization_rebuild(dbName, tableName, txnId) + send_get_lock_materialization_rebuild(dbName, tableName, txnId) + return recv_get_lock_materialization_rebuild() + end + + def send_get_lock_materialization_rebuild(dbName, tableName, txnId) + send_message('get_lock_materialization_rebuild', Get_lock_materialization_rebuild_args, :dbName => dbName, :tableName => tableName, :txnId => txnId) + end + + def recv_get_lock_materialization_rebuild() + result = receive_message(Get_lock_materialization_rebuild_result) + return result.success unless result.success.nil? + raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_lock_materialization_rebuild failed: unknown result') + end + + def heartbeat_lock_materialization_rebuild(dbName, tableName, txnId) + send_heartbeat_lock_materialization_rebuild(dbName, tableName, txnId) + return recv_heartbeat_lock_materialization_rebuild() + end + + def send_heartbeat_lock_materialization_rebuild(dbName, tableName, txnId) + send_message('heartbeat_lock_materialization_rebuild', Heartbeat_lock_materialization_rebuild_args, :dbName => dbName, :tableName => tableName, :txnId => txnId) + end + + def recv_heartbeat_lock_materialization_rebuild() + result = receive_message(Heartbeat_lock_materialization_rebuild_result) + return result.success unless result.success.nil? + raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'heartbeat_lock_materialization_rebuild failed: unknown result') + end + end class Processor < ::FacebookService::Processor @@ -5875,6 +5905,20 @@ module ThriftHiveMetastore write_result(result, oprot, 'get_serde', seqid) end + def process_get_lock_materialization_rebuild(seqid, iprot, oprot) + args = read_args(iprot, Get_lock_materialization_rebuild_args) + result = Get_lock_materialization_rebuild_result.new() + result.success = @handler.get_lock_materialization_rebuild(args.dbName, args.tableName, args.txnId) + write_result(result, oprot, 'get_lock_materialization_rebuild', seqid) + end + + def process_heartbeat_lock_materialization_rebuild(seqid, iprot, oprot) + args = read_args(iprot, Heartbeat_lock_materialization_rebuild_args) + result = Heartbeat_lock_materialization_rebuild_result.new() + result.success = @handler.heartbeat_lock_materialization_rebuild(args.dbName, args.tableName, args.txnId) + write_result(result, oprot, 'heartbeat_lock_materialization_rebuild', seqid) + end + end # HELPER FUNCTIONS AND STRUCTURES @@ -13301,5 +13345,77 @@ module ThriftHiveMetastore ::Thrift::Struct.generate_accessors self end + class Get_lock_materialization_rebuild_args + include ::Thrift::Struct, ::Thrift::Struct_Union + DBNAME = 1 + TABLENAME = 2 + TXNID = 3 + + FIELDS = { + DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'}, + TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'}, + TXNID => {:type => ::Thrift::Types::I64, :name => 'txnId'} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Get_lock_materialization_rebuild_result + include ::Thrift::Struct, ::Thrift::Struct_Union + SUCCESS = 0 + + FIELDS = { + SUCCESS => {:type => ::Thrift::Types::STRUCT, :name => 'success', :class => ::LockResponse} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Heartbeat_lock_materialization_rebuild_args + include ::Thrift::Struct, ::Thrift::Struct_Union + DBNAME = 1 + TABLENAME = 2 + TXNID = 3 + + FIELDS = { + DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'}, + TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'}, + TXNID => {:type => ::Thrift::Types::I64, :name => 'txnId'} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Heartbeat_lock_materialization_rebuild_result + include ::Thrift::Struct, ::Thrift::Struct_Union + SUCCESS = 0 + + FIELDS = { + SUCCESS => {:type => ::Thrift::Types::BOOL, :name => 'success'} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + end diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index a2fe7d72eb..4381b3b87f 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -8367,6 +8367,18 @@ public SerDeInfo get_serde(GetSerdeRequest rqst) throws TException { endFunction("get_serde", serde != null, ex); } } + + @Override + public LockResponse get_lock_materialization_rebuild(String dbName, String tableName, long txnId) + throws TException { + return MaterializationsRebuildLockHandler.get().lockResource(dbName, tableName, txnId); + } + + @Override + public boolean heartbeat_lock_materialization_rebuild(String dbName, String tableName, long txnId) + throws TException { + return MaterializationsRebuildLockHandler.get().refreshLockResource(dbName, tableName, txnId); + } } private static IHMSHandler newRetryingHMSHandler(IHMSHandler baseHandler, Configuration conf) diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index ebbf465244..95a3767c65 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -3127,4 +3127,14 @@ private short shrinkMaxtoShort(int max) { else if (max <= Short.MAX_VALUE) return (short)max; else return Short.MAX_VALUE; } + + @Override + public LockResponse lockMaterializationRebuild(String dbName, String tableName, long txnId) throws TException { + return client.get_lock_materialization_rebuild(dbName, tableName, txnId); + } + + @Override + public boolean heartbeatLockMaterializationRebuild(String dbName, String tableName, long txnId) throws TException { + return client.heartbeat_lock_materialization_rebuild(dbName, tableName, txnId); + } } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index b2c40c25f2..98674cf7de 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -3613,4 +3613,25 @@ void createOrDropTriggerToPoolMapping(String resourcePlanName, String triggerNam * @throws TException general thrift error */ SerDeInfo getSerDe(String serDeName) throws TException; + + /** + * Acquire the materialization rebuild lock for a given view. We need to specify the fully + * qualified name of the materialized view and the open transaction ID so we can identify + * uniquely the lock. + * @param dbName db name for the materialized view + * @param tableName table name for the materialized view + * @param txnId transaction id for the rebuild + * @return the response from the metastore, where the lock id is equal to the txn id and + * the status can be either ACQUIRED or NOT ACQUIRED + */ + LockResponse lockMaterializationRebuild(String dbName, String tableName, long txnId) throws TException; + + /** + * Method to refresh the acquisition of a given materialization rebuild lock. + * @param dbName db name for the materialized view + * @param tableName table name for the materialized view + * @param txnId transaction id for the rebuild + * @return true if the lock could be renewed, false otherwise + */ + boolean heartbeatLockMaterializationRebuild(String dbName, String tableName, long txnId) throws TException; } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationInvalidationInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationInvalidationInfo.java deleted file mode 100644 index 3d774071c2..0000000000 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationInvalidationInfo.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore; - -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.hadoop.hive.metastore.api.Materialization; -import org.apache.hadoop.hive.metastore.api.Table; - -/** - * Contains information about the invalidation of a materialization, - * including the materialization name, the tables that it uses, and - * the invalidation time, i.e., the first moment t0 after the - * materialization was created at which one of the tables that it uses - * was modified. - */ -@SuppressWarnings("serial") -public class MaterializationInvalidationInfo extends Materialization { - - private AtomicLong invalidationTime; - - public MaterializationInvalidationInfo(Set tablesUsed, String validTxnList) { - super(tablesUsed, 0); - this.setValidTxnList(validTxnList); - this.invalidationTime = new AtomicLong(0); - } - - public boolean compareAndSetInvalidationTime(long expect, long update) { - boolean success = invalidationTime.compareAndSet(expect, update); - if (success) { - super.setInvalidationTime(update); - } - return success; - } - - public long getInvalidationTime() { - return invalidationTime.get(); - } - - public void setInvalidationTime(long invalidationTime) { - throw new UnsupportedOperationException("You should call compareAndSetInvalidationTime instead"); - } - -} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsInvalidationCache.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsInvalidationCache.java index 80cb1de75e..99c5abcf59 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsInvalidationCache.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsInvalidationCache.java @@ -26,15 +26,17 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.ValidReadTxnList; -import org.apache.hadoop.hive.common.ValidTxnList; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; +import org.apache.hadoop.hive.common.ValidTxnWriteIdList; +import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.metastore.api.BasicTxnInfo; +import org.apache.hadoop.hive.metastore.api.LockResponse; import org.apache.hadoop.hive.metastore.api.Materialization; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Table; @@ -65,10 +67,12 @@ /* Key is the database name. Each value is a map from the unique view qualified name to * the materialization invalidation info. This invalidation object contains information - * such as the tables used by the materialized view or the invalidation time, i.e., first - * modification of the tables used by materialized view after the view was created. */ - private final ConcurrentMap> materializations = - new ConcurrentHashMap>(); + * such as the tables used by the materialized view, whether there was any update or + * delete in the source tables since the materialized view was created or rebuilt, + * or the invalidation time, i.e., first modification of the tables used by materialized + * view after the view was created. */ + private final ConcurrentMap> materializations = + new ConcurrentHashMap<>(); /* * Key is a qualified table name. The value is a (sorted) tree map (supporting concurrent @@ -77,7 +81,10 @@ * materialization. */ private final ConcurrentMap> tableModifications = - new ConcurrentHashMap>(); + new ConcurrentHashMap<>(); + + private final ConcurrentMap> updateDeleteTableModifications = + new ConcurrentHashMap<>(); /* Whether the cache has been initialized or not. */ private boolean initialized; @@ -188,9 +195,9 @@ private void addMaterializedView(String dbName, String tableName, Set ta return; } // We are going to create the map for each view in the given database - ConcurrentMap cq = - new ConcurrentHashMap(); - final ConcurrentMap prevCq = materializations.putIfAbsent( + ConcurrentMap cq = + new ConcurrentHashMap(); + final ConcurrentMap prevCq = materializations.putIfAbsent( dbName, cq); if (prevCq != null) { cq = prevCq; @@ -204,13 +211,15 @@ private void addMaterializedView(String dbName, String tableName, Set ta } if (opType == OpType.CREATE || opType == OpType.ALTER) { // You store the materialized view - cq.put(tableName, new MaterializationInvalidationInfo(tablesUsed, validTxnList)); + Materialization materialization = new Materialization(tablesUsed); + materialization.setValidTxnList(validTxnList); + cq.put(tableName, materialization); } else { - ValidTxnList txnList = new ValidReadTxnList(validTxnList); + ValidTxnWriteIdList txnList = new ValidTxnWriteIdList(validTxnList); for (String qNameTableUsed : tablesUsed) { + ValidWriteIdList tableTxnList = txnList.getTableValidWriteIdList(qNameTableUsed); // First we insert a new tree set to keep table modifications, unless it already exists - ConcurrentSkipListMap modificationsTree = - new ConcurrentSkipListMap(); + ConcurrentSkipListMap modificationsTree = new ConcurrentSkipListMap<>(); final ConcurrentSkipListMap prevModificationsTree = tableModifications.putIfAbsent( qNameTableUsed, modificationsTree); if (prevModificationsTree != null) { @@ -222,7 +231,7 @@ private void addMaterializedView(String dbName, String tableName, Set ta try { String[] names = qNameTableUsed.split("\\."); BasicTxnInfo e = handler.getTxnHandler().getFirstCompletedTransactionForTableAfterCommit( - names[0], names[1], txnList); + names[0], names[1], tableTxnList); if (!e.isIsnull()) { modificationsTree.put(e.getTxnid(), e.getTime()); // We do not need to do anything more for current table, as we detected @@ -236,7 +245,9 @@ private void addMaterializedView(String dbName, String tableName, Set ta } } // For LOAD, you only add it if it does exist as you might be loading an outdated MV - cq.putIfAbsent(tableName, new MaterializationInvalidationInfo(tablesUsed, validTxnList)); + Materialization materialization = new Materialization(tablesUsed); + materialization.setValidTxnList(validTxnList); + cq.putIfAbsent(tableName, materialization); } if (LOG.isDebugEnabled()) { LOG.debug("Cached materialized view for rewriting in invalidation cache: " + @@ -249,7 +260,7 @@ private void addMaterializedView(String dbName, String tableName, Set ta * invalidation for the MVs that use that table. */ public void notifyTableModification(String dbName, String tableName, - long txnId, long newModificationTime) { + long txnId, long newModificationTime, boolean isUpdateDelete) { if (disable) { // Nothing to do return; @@ -258,8 +269,18 @@ public void notifyTableModification(String dbName, String tableName, LOG.debug("Notification for table {} in database {} received -> id: {}, time: {}", tableName, dbName, txnId, newModificationTime); } - ConcurrentSkipListMap modificationsTree = - new ConcurrentSkipListMap(); + if (isUpdateDelete) { + // We update first the update/delete modifications record + ConcurrentSkipListSet modificationsSet = new ConcurrentSkipListSet<>(); + final ConcurrentSkipListSet prevModificationsSet = + updateDeleteTableModifications.putIfAbsent(Warehouse.getQualifiedName(dbName, tableName), + modificationsSet); + if (prevModificationsSet != null) { + modificationsSet = prevModificationsSet; + } + modificationsSet.add(txnId); + } + ConcurrentSkipListMap modificationsTree = new ConcurrentSkipListMap<>(); final ConcurrentSkipListMap prevModificationsTree = tableModifications.putIfAbsent(Warehouse.getQualifiedName(dbName, tableName), modificationsTree); if (prevModificationsTree != null) { @@ -293,30 +314,21 @@ public void dropMaterializedView(String dbName, String tableName) { if (materializations.get(dbName) != null) { ImmutableMap.Builder m = ImmutableMap.builder(); for (String materializationName : materializationNames) { - MaterializationInvalidationInfo materialization = + Materialization materialization = materializations.get(dbName).get(materializationName); if (materialization == null) { LOG.debug("Materialization {} skipped as there is no information " + "in the invalidation cache about it", materializationName); continue; } - long invalidationTime = getInvalidationTime(materialization); - // We need to check whether previous value is zero, as data modification - // in another table used by the materialized view might have modified - // the value too - boolean modified = materialization.compareAndSetInvalidationTime(0L, invalidationTime); - while (!modified) { - long currentInvalidationTime = materialization.getInvalidationTime(); - if (invalidationTime < currentInvalidationTime) { - // It was set by other table modification, but it was after this table modification - // hence we need to set it - modified = materialization.compareAndSetInvalidationTime(currentInvalidationTime, invalidationTime); - } else { - // Nothing to do - modified = true; - } - } - m.put(materializationName, materialization); + // We create a deep copy of the materialization, as we need to set the time + // and whether any update/delete operation happen on the tables that it uses + // since it was created. + Materialization materializationCopy = new Materialization( + materialization.getTablesUsed()); + materializationCopy.setValidTxnList(materialization.getValidTxnList()); + enrichWithInvalidationInfo(materializationCopy); + m.put(materializationName, materializationCopy); } Map result = m.build(); if (LOG.isDebugEnabled()) { @@ -327,50 +339,65 @@ public void dropMaterializedView(String dbName, String tableName) { return ImmutableMap.of(); } - private long getInvalidationTime(MaterializationInvalidationInfo materialization) { - String txnListString = materialization.getValidTxnList(); - if (txnListString == null) { + private void enrichWithInvalidationInfo(Materialization materialization) { + String materializationTxnListString = materialization.getValidTxnList(); + if (materializationTxnListString == null) { // This can happen when the materialization was created on non-transactional tables - return Long.MIN_VALUE; + materialization.setInvalidationTime(Long.MIN_VALUE); + return; } // We will obtain the modification time as follows. // First, we obtain the first element after high watermark (if any) // Then, we iterate through the elements from min open txn till high // watermark, updating the modification time after creation if needed - ValidTxnList txnList = new ValidReadTxnList(txnListString); + ValidTxnWriteIdList materializationTxnList = new ValidTxnWriteIdList(materializationTxnListString); long firstModificationTimeAfterCreation = 0L; + boolean containsUpdateDelete = false; for (String qNameTableUsed : materialization.getTablesUsed()) { - final Entry tn = tableModifications.get(qNameTableUsed) - .higherEntry(txnList.getHighWatermark()); + final ValidWriteIdList tableMaterializationTxnList = + materializationTxnList.getTableValidWriteIdList(qNameTableUsed); + + final ConcurrentSkipListMap usedTableModifications = + tableModifications.get(qNameTableUsed); + final ConcurrentSkipListSet usedUDTableModifications = + updateDeleteTableModifications.get(qNameTableUsed); + final Entry tn = usedTableModifications.higherEntry(tableMaterializationTxnList.getHighWatermark()); if (tn != null) { if (firstModificationTimeAfterCreation == 0L || tn.getValue() < firstModificationTimeAfterCreation) { firstModificationTimeAfterCreation = tn.getValue(); } + // Check if there was any update/delete after creation + containsUpdateDelete = usedUDTableModifications != null && + !usedUDTableModifications.tailSet(tableMaterializationTxnList.getHighWatermark(), false).isEmpty(); } // Min open txn might be null if there were no open transactions // when this transaction was being executed - if (txnList.getMinOpenTxn() != null) { + if (tableMaterializationTxnList.getMinOpenWriteId() != null) { // Invalid transaction list is sorted int pos = 0; - for (Map.Entry t : tableModifications.get(qNameTableUsed) - .subMap(txnList.getMinOpenTxn(), txnList.getHighWatermark()).entrySet()) { - while (pos < txnList.getInvalidTransactions().length && - txnList.getInvalidTransactions()[pos] != t.getKey()) { + for (Map.Entry t : usedTableModifications + .subMap(tableMaterializationTxnList.getMinOpenWriteId(), tableMaterializationTxnList.getHighWatermark()).entrySet()) { + while (pos < tableMaterializationTxnList.getInvalidWriteIds().length && + tableMaterializationTxnList.getInvalidWriteIds()[pos] != t.getKey()) { pos++; } - if (pos >= txnList.getInvalidTransactions().length) { + if (pos >= tableMaterializationTxnList.getInvalidWriteIds().length) { break; } if (firstModificationTimeAfterCreation == 0L || t.getValue() < firstModificationTimeAfterCreation) { firstModificationTimeAfterCreation = t.getValue(); } + containsUpdateDelete = containsUpdateDelete || + (usedUDTableModifications != null && usedUDTableModifications.contains(t.getKey())); } } } - return firstModificationTimeAfterCreation; + + materialization.setInvalidationTime(firstModificationTimeAfterCreation); + materialization.setSourceTablesUpdateDeleteModified(containsUpdateDelete); } private enum OpType { @@ -395,16 +422,17 @@ public long cleanup(long minTime) { // We execute the cleanup in two steps // First we gather all the transactions that need to be kept final Multimap keepTxnInfos = HashMultimap.create(); - for (Map.Entry> e : materializations.entrySet()) { - for (MaterializationInvalidationInfo m : e.getValue().values()) { - ValidTxnList txnList = new ValidReadTxnList(m.getValidTxnList()); + for (Map.Entry> e : materializations.entrySet()) { + for (Materialization m : e.getValue().values()) { + ValidTxnWriteIdList txnList = new ValidTxnWriteIdList(m.getValidTxnList()); boolean canBeDeleted = false; String currentTableForInvalidatingTxn = null; long currentInvalidatingTxnId = 0L; long currentInvalidatingTxnTime = 0L; for (String qNameTableUsed : m.getTablesUsed()) { + ValidWriteIdList tableTxnList = txnList.getTableValidWriteIdList(qNameTableUsed); final Entry tn = tableModifications.get(qNameTableUsed) - .higherEntry(txnList.getHighWatermark()); + .higherEntry(tableTxnList.getHighWatermark()); if (tn != null) { if (currentInvalidatingTxnTime == 0L || tn.getValue() < currentInvalidatingTxnTime) { @@ -424,16 +452,16 @@ public long cleanup(long minTime) { currentInvalidatingTxnTime = tn.getValue(); } } - if (txnList.getMinOpenTxn() != null) { + if (tableTxnList.getMinOpenWriteId() != null) { // Invalid transaction list is sorted int pos = 0; for (Entry t : tableModifications.get(qNameTableUsed) - .subMap(txnList.getMinOpenTxn(), txnList.getHighWatermark()).entrySet()) { - while (pos < txnList.getInvalidTransactions().length && - txnList.getInvalidTransactions()[pos] != t.getKey()) { + .subMap(tableTxnList.getMinOpenWriteId(), tableTxnList.getHighWatermark()).entrySet()) { + while (pos < tableTxnList.getInvalidWriteIds().length && + tableTxnList.getInvalidWriteIds()[pos] != t.getKey()) { pos++; } - if (pos >= txnList.getInvalidTransactions().length) { + if (pos >= tableTxnList.getInvalidWriteIds().length) { break; } if (currentInvalidatingTxnTime == 0L || @@ -462,6 +490,7 @@ public long cleanup(long minTime) { long removed = 0L; for (Entry> e : tableModifications.entrySet()) { Collection c = keepTxnInfos.get(e.getKey()); + ConcurrentSkipListSet updateDeleteForTable = updateDeleteTableModifications.get(e.getKey()); for (Iterator> it = e.getValue().entrySet().iterator(); it.hasNext();) { Entry v = it.next(); // We need to check again the time because some of the transactions might not be explored @@ -472,6 +501,9 @@ public long cleanup(long minTime) { LOG.debug("Transaction removed from cache for table {} -> id: {}, time: {}", e.getKey(), v.getKey(), v.getValue()); } + if (updateDeleteForTable != null) { + updateDeleteForTable.remove(v.getKey()); + } it.remove(); removed++; } @@ -480,4 +512,23 @@ public long cleanup(long minTime) { return removed; } + /** + * Checks whether the given materialization exists in the invalidation cache. + * @param dbName the database name for the materialization + * @param tblName the table name for the materialization + * @return true if we have information about the materialization in the cache, + * false otherwise + */ + public boolean containsMaterialization(String dbName, String tblName) { + if (disable || dbName == null || tblName == null) { + return false; + } + ConcurrentMap dbMaterializations = materializations.get(dbName); + if (dbMaterializations == null || dbMaterializations.get(tblName) == null) { + // This is a table + return false; + } + return true; + } + } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java new file mode 100644 index 0000000000..8ca9ede074 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.TimeUnit; + +/** + * Cleaner for the {@link MaterializationsRebuildLockHandler}. It removes outdated locks + * in the intervals specified by the input property. + */ +public class MaterializationsRebuildLockCleanerTask implements MetastoreTaskThread { + private static final Logger LOG = LoggerFactory.getLogger(MaterializationsRebuildLockCleanerTask.class); + + private Configuration conf; + + @Override + public long runFrequency(TimeUnit unit) { + return MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.TXN_TIMEOUT, unit) / 2; + } + + @Override + public void setConf(Configuration configuration) { + conf = configuration; + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void run() { + long removedCnt = MaterializationsRebuildLockHandler.get().cleanupResourceLocks( + MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.TXN_TIMEOUT, TimeUnit.MILLISECONDS)); + if (removedCnt > 0) { + if (LOG.isDebugEnabled()) { + LOG.info("Number of materialization locks deleted: " + removedCnt); + } + } + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockHandler.java new file mode 100644 index 0000000000..dd31226dca --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockHandler.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.hive.metastore.api.LockResponse; +import org.apache.hadoop.hive.metastore.api.LockState; +import org.apache.hadoop.hive.metastore.api.MetaException; + +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +/** + * This is a lock handler implementation for the materializations rebuild. + * It is lightweight: it does not persist any information to metastore db. + * Its states are as follows: + * 1) request lock -> 2) ACQUIRED -> 4) COMMIT_READY -> 6) release lock + * -> 5) EXPIRED -> + * -> 3) NOT_ACQUIRED + * First, the rebuild operation will ACQUIRE the lock. If other rebuild + * operation for the same operation is already running, we lock status + * will be NOT_ACQUIRED. + * Before committing the rebuild, the txn handler will signal the handler + * that it is ready to commit the resource (move state to COMMIT_READY). + * We make sure the lock is still available before moving to the new state. + * A lock will not be able to expire when it is in COMMIT_READY state. + * The unlock method is always call by the txn handler, no matter whether + * the transaction succeeds or not, e.g., due to an Exception. + * From ACQUIRED, locks can be also moved to EXPIRED state when they + * expire. From EXPIRED, they can only be released. + */ +public class MaterializationsRebuildLockHandler { + + /* Singleton */ + private static final MaterializationsRebuildLockHandler SINGLETON = new MaterializationsRebuildLockHandler(); + + private final ConcurrentMap locks = new ConcurrentHashMap<>(); + + private MaterializationsRebuildLockHandler() { + } + + /** + * Get instance of MaterializationsRebuildLockHandler. + * + * @return the singleton + */ + public static MaterializationsRebuildLockHandler get() { + return SINGLETON; + } + + /** + * Lock materialized view (first step for rebuild). Response contains a lock id + * that corresponds to the input transaction id, and whether the lock was + * ACQUIRED or NOT_ACQUIRED. + * @param dbName the db name of the materialization + * @param tableName the table name of the materialization + * @param txnId the transaction id for the rebuild + * @return the response to the lock request + */ + public LockResponse lockResource(String dbName, String tableName, long txnId) { + final ResourceLock prevResourceLock = locks.putIfAbsent( + Warehouse.getQualifiedName(dbName, tableName), + new ResourceLock(txnId, System.nanoTime(), State.ACQUIRED)); + if (prevResourceLock != null) { + return new LockResponse(txnId, LockState.NOT_ACQUIRED); + } + return new LockResponse(txnId, LockState.ACQUIRED); + } + + /** + * Moves from ACQUIRED state to COMMIT_READY. + * @param dbName the db name of the materialization + * @param tableName the table name of the materialization + * @param txnId the transaction id for the rebuild + * @return true if the lock was still active and we could move the materialization + * to COMMIT_READY state, false otherwise + */ + public boolean readyToCommitResource(String dbName, String tableName, long txnId) { + final ResourceLock prevResourceLock = locks.get(Warehouse.getQualifiedName(dbName, tableName)); + if (prevResourceLock == null || prevResourceLock.txnId != txnId) { + // Lock was outdated and it was removed (then maybe another transaction picked it up) + return false; + } + return prevResourceLock.state.compareAndSet(State.ACQUIRED, State.COMMIT_READY); + } + + /** + * Heartbeats a certain lock and refreshes its timer. + * @param dbName the db name of the materialization + * @param tableName the table name of the materialization + * @param txnId the transaction id for the rebuild + * @throws MetaException + */ + public boolean refreshLockResource(String dbName, String tableName, long txnId) { + final ResourceLock prevResourceLock = locks.get(Warehouse.getQualifiedName(dbName, tableName)); + if (prevResourceLock == null || prevResourceLock.txnId != txnId || + prevResourceLock.state.get() != State.ACQUIRED) { + // Lock was outdated and it was removed (then maybe another transaction picked it up) + // or changed its state + return false; + } + prevResourceLock.lastHeartBeatTime.set(System.currentTimeMillis()); + return true; + } + + /** + * Releases a certain lock. + * @param dbName the db name of the materialization + * @param tableName the table name of the materialization + * @param txnId the transaction id for the rebuild + * @return true if the lock could be released properly, false otherwise + * @throws MetaException + */ + public boolean unlockResource(String dbName, String tableName, long txnId) { + final String fullyQualifiedName = Warehouse.getQualifiedName(dbName, tableName); + final ResourceLock prevResourceLock = locks.get(fullyQualifiedName); + if (prevResourceLock == null || prevResourceLock.txnId != txnId) { + return false; + } + return locks.remove(fullyQualifiedName, prevResourceLock); + } + + /** + * Method that removes from the handler those locks that have expired. + * @param timeout time after which we consider the locks to have expired + * @throws MetaException + */ + public long cleanupResourceLocks(long timeout) { + long removed = 0L; + final long currentTime = System.currentTimeMillis(); + for (Iterator> it = locks.entrySet().iterator(); it.hasNext();) { + final ResourceLock resourceLock = it.next().getValue(); + if (currentTime - resourceLock.lastHeartBeatTime.get() > timeout) { + if (resourceLock.state.compareAndSet(State.ACQUIRED, State.EXPIRED)) { + it.remove(); + removed++; + } + } + } + return removed; + } + + /** + * This class represents a lock that consists of transaction id, + * last refresh time, and state. + */ + private class ResourceLock { + final long txnId; + final AtomicLong lastHeartBeatTime; + final AtomicStateEnum state; + + ResourceLock(long txnId, long lastHeartBeatTime, State state) { + this.txnId = txnId; + this.lastHeartBeatTime = new AtomicLong(lastHeartBeatTime); + this.state = new AtomicStateEnum(state); + } + } + + private enum State { + // This is the initial state for a lock + ACQUIRED, + // This means that the lock is being committed at this instant, hence + // the cleaner should not remove it even if it times out. If transaction + // fails, the finally clause will remove the lock + COMMIT_READY, + // This means that the lock is ready to be cleaned, hence it cannot + // be committed anymore + EXPIRED; + } + + /** + * Wrapper class around State enum to make its operations atomic. + */ + private class AtomicStateEnum { + private final AtomicReference ref; + + public AtomicStateEnum(final State initialValue) { + this.ref = new AtomicReference(initialValue); + } + + public void set(final State newValue) { + this.ref.set(newValue); + } + + public State get() { + return this.ref.get(); + } + + public State getAndSet(final State newValue) { + return this.ref.getAndSet(newValue); + } + + public boolean compareAndSet(final State expect, final State update) { + return this.ref.compareAndSet(expect, update); + } + } + +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 940a1bf276..f007261daf 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader; import org.apache.hadoop.hive.metastore.HiveAlterHandler; import org.apache.hadoop.hive.metastore.MaterializationsCacheCleanerTask; +import org.apache.hadoop.hive.metastore.MaterializationsRebuildLockCleanerTask; import org.apache.hadoop.hive.metastore.MetastoreTaskThread; import org.apache.hadoop.hive.metastore.events.EventCleanerTask; import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager; @@ -728,7 +729,8 @@ public static ConfVars getMetaConf(String name) { TASK_THREADS_ALWAYS("metastore.task.threads.always", "metastore.task.threads.always", EventCleanerTask.class.getName() + "," + "org.apache.hadoop.hive.metastore.repl.DumpDirCleanerTask" + "," + - MaterializationsCacheCleanerTask.class.getName(), + MaterializationsCacheCleanerTask.class.getName() + "," + + MaterializationsRebuildLockCleanerTask.class.getName(), "Comma separated list of tasks that will be started in separate threads. These will " + "always be started, regardless of whether the metastore is running in embedded mode " + "or in server mode. They must implement " + MetastoreTaskThread.class.getName()), diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index a79242b1a1..9256b7a323 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -63,9 +63,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.ValidReadTxnList; import org.apache.hadoop.hive.common.ValidTxnList; +import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.common.classification.RetrySemantics; import org.apache.hadoop.hive.metastore.DatabaseProduct; import org.apache.hadoop.hive.metastore.MaterializationsInvalidationCache; +import org.apache.hadoop.hive.metastore.MaterializationsRebuildLockHandler; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier; import org.apache.hadoop.hive.metastore.TransactionalMetaStoreEventListener; @@ -854,9 +856,18 @@ public void abortTxns(AbortTxnsRequest rqst) throws NoSuchTxnException, MetaExce @Override @RetrySemantics.Idempotent("No-op if already committed") public void commitTxn(CommitTxnRequest rqst) - throws NoSuchTxnException, TxnAbortedException, MetaException { + throws NoSuchTxnException, TxnAbortedException, MetaException { + MaterializationsRebuildLockHandler materializationsRebuildLockHandler = + MaterializationsRebuildLockHandler.get(); + String fullyQualifiedName = null; + String dbName = null; + String tblName = null; + long writeId = 0L; + long timestamp = 0L; + boolean isUpdateDelete = false; long txnid = rqst.getTxnid(); long sourceTxnId = -1; + try { Connection dbConn = null; Statement stmt = null; @@ -905,6 +916,7 @@ public void commitTxn(CommitTxnRequest rqst) quoteChar(OpertaionType.UPDATE.sqlConst) + "," + quoteChar(OpertaionType.DELETE.sqlConst) + ")"; rs = stmt.executeQuery(sqlGenerator.addLimitClause(1, "tc_operation_type " + conflictSQLSuffix)); if (rs.next()) { + isUpdateDelete = true; close(rs); //if here it means currently committing txn performed update/delete and we should check WW conflict /** @@ -1007,6 +1019,17 @@ public void commitTxn(CommitTxnRequest rqst) LOG.info("Expected to move at least one record from txn_components to " + "completed_txn_components when committing txn! " + JavaUtils.txnIdToString(txnid)); } + // Obtain information that we need to update registry + s = "select ctc_database, ctc_table, ctc_writeid, ctc_timestamp from COMPLETED_TXN_COMPONENTS where ctc_txnid = " + txnid; + LOG.debug("Going to extract table modification information for invalidation cache <" + s + ">"); + rs = stmt.executeQuery(s); + if (rs.next()) { + dbName = rs.getString(1); + tblName = rs.getString(2); + fullyQualifiedName = Warehouse.getQualifiedName(dbName, tblName); + writeId = rs.getLong(3); + timestamp = rs.getTimestamp(4, Calendar.getInstance(TimeZone.getTimeZone("UTC"))).getTime(); + } s = "delete from TXN_COMPONENTS where tc_txnid = " + txnid; LOG.debug("Going to execute update <" + s + ">"); modCount = stmt.executeUpdate(s); @@ -1021,16 +1044,6 @@ public void commitTxn(CommitTxnRequest rqst) modCount = stmt.executeUpdate(s); LOG.info("Removed committed transaction: (" + txnid + ") from MIN_HISTORY_LEVEL"); - // Update registry with modifications - s = "select ctc_database, ctc_table, ctc_timestamp from COMPLETED_TXN_COMPONENTS where ctc_txnid = " + txnid; - rs = stmt.executeQuery(s); - if (rs.next()) { - LOG.debug("Going to register table modification in invalidation cache <" + s + ">"); - MaterializationsInvalidationCache.get().notifyTableModification( - rs.getString(1), rs.getString(2), txnid, - rs.getTimestamp(3, Calendar.getInstance(TimeZone.getTimeZone("UTC"))).getTime()); - } - if (rqst.isSetReplPolicy()) { s = "delete from REPL_TXN_MAP where RTM_SRC_TXN_ID = " + sourceTxnId + " and RTM_REPL_POLICY = " + quoteString(rqst.getReplPolicy()); @@ -1043,10 +1056,20 @@ public void commitTxn(CommitTxnRequest rqst) EventMessage.EventType.COMMIT_TXN, new CommitTxnEvent(txnid, dbConn, sqlGenerator)); } - close(rs); - + MaterializationsInvalidationCache materializationsInvalidationCache = + MaterializationsInvalidationCache.get(); + if (materializationsInvalidationCache.containsMaterialization(dbName, tblName) && + !materializationsRebuildLockHandler.readyToCommitResource(dbName, tblName, txnid)) { + throw new MetaException( + "Another process is rebuilding the materialized view " + fullyQualifiedName); + } LOG.debug("Going to commit"); + close(rs); dbConn.commit(); + + // Update registry with modifications + materializationsInvalidationCache.notifyTableModification( + dbName, tblName, writeId, timestamp, isUpdateDelete); } catch (SQLException e) { LOG.debug("Going to rollback"); rollbackDBConn(dbConn); @@ -1057,6 +1080,9 @@ public void commitTxn(CommitTxnRequest rqst) close(commitIdRs); close(lockHandle, stmt, dbConn); unlockInternal(); + if (fullyQualifiedName != null) { + materializationsRebuildLockHandler.unlockResource(dbName, tblName, txnid); + } } } catch (RetryException e) { commitTxn(rqst); @@ -1384,9 +1410,9 @@ public void performWriteSetGC() { @Override @RetrySemantics.ReadOnly public BasicTxnInfo getFirstCompletedTransactionForTableAfterCommit( - String inputDbName, String inputTableName, ValidTxnList txnList) + String inputDbName, String inputTableName, ValidWriteIdList txnList) throws MetaException { - final List openTxns = Arrays.asList(ArrayUtils.toObject(txnList.getInvalidTransactions())); + final List openTxns = Arrays.asList(ArrayUtils.toObject(txnList.getInvalidWriteIds())); Connection dbConn = null; Statement stmt = null; @@ -1395,12 +1421,12 @@ public BasicTxnInfo getFirstCompletedTransactionForTableAfterCommit( dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); stmt = dbConn.createStatement(); stmt.setMaxRows(1); - String s = "select ctc_timestamp, ctc_txnid, ctc_database, ctc_table " + String s = "select ctc_timestamp, ctc_writeid, ctc_database, ctc_table " + "from COMPLETED_TXN_COMPONENTS " + "where ctc_database=" + quoteString(inputDbName) + " and ctc_table=" + quoteString(inputTableName) - + " and ctc_txnid > " + txnList.getHighWatermark() - + (txnList.getInvalidTransactions().length == 0 ? - " " : " or ctc_txnid IN(" + StringUtils.join(",", openTxns) + ") ") + + " and ctc_writeid > " + txnList.getHighWatermark() + + (txnList.getInvalidWriteIds().length == 0 ? + " " : " or ctc_writeid IN(" + StringUtils.join(",", openTxns) + ") ") + "order by ctc_timestamp asc"; if (LOG.isDebugEnabled()) { LOG.debug("Going to execute query <" + s + ">"); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java index e72d327892..6d8b8456a7 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.hive.common.ValidTxnList; +import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.common.classification.RetrySemantics; import org.apache.hadoop.hive.metastore.api.*; @@ -123,7 +124,7 @@ void commitTxn(CommitTxnRequest rqst) */ @RetrySemantics.Idempotent public BasicTxnInfo getFirstCompletedTransactionForTableAfterCommit( - String inputDbName, String inputTableName, ValidTxnList txnList) + String inputDbName, String inputTableName, ValidWriteIdList txnList) throws MetaException; /** * Gets the list of valid write ids for the given table wrt to current txn diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift index 7450439885..612afe1cc0 100644 --- a/standalone-metastore/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift @@ -1227,7 +1227,8 @@ struct TableMeta { struct Materialization { 1: required set tablesUsed; 2: optional string validTxnList - 3: required i64 invalidationTime; + 3: optional i64 invalidationTime; + 4: optional bool sourceTablesUpdateDeleteModified; } // Data types for workload management. @@ -2163,6 +2164,8 @@ service ThriftHiveMetastore extends fb303.FacebookService void add_serde(1: SerDeInfo serde) throws(1:AlreadyExistsException o1, 2:MetaException o2) SerDeInfo get_serde(1: GetSerdeRequest rqst) throws(1:NoSuchObjectException o1, 2:MetaException o2) + LockResponse get_lock_materialization_rebuild(1: string dbName, 2: string tableName, 3: i64 txnId) + bool heartbeat_lock_materialization_rebuild(1: string dbName, 2: string tableName, 3: i64 txnId) } // * Note about the DDL_TIME: When creating or altering a table or a partition, diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 7d372627a4..ecddc7bf28 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -3321,4 +3321,16 @@ public void addSerDe(SerDeInfo serDeInfo) throws TException { public SerDeInfo getSerDe(String serDeName) throws TException { throw new UnsupportedOperationException(); } + + /** {@inheritDoc} */ + @Override + public LockResponse lockMaterializationRebuild(String dbName, String tableName, long txnId) throws TException { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override + public boolean heartbeatLockMaterializationRebuild(String dbName, String tableName, long txnId) throws TException { + throw new UnsupportedOperationException(); + } } diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java index 7a871e1458..8debccefb3 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java @@ -65,16 +65,16 @@ public void testCleanerScenario1() throws Exception { // This is a dummy test, invalidation cache is not supposed to // record any information. MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 1, 1); + DB_NAME, TBL_NAME_1, 1, 1, false); int id = 2; BasicTxnInfo txn2 = createTxnInfo(DB_NAME, TBL_NAME_1, id); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, id, id); + DB_NAME, TBL_NAME_1, id, id, false); // Create tbl2 (nothing to do) id = 3; BasicTxnInfo txn3 = createTxnInfo(DB_NAME, TBL_NAME_1, id); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, id, id); + DB_NAME, TBL_NAME_2, id, id, false); // Cleanup (current = 4, duration = 4) -> Does nothing long removed = MaterializationsInvalidationCache.get().cleanup(0L); Assert.assertEquals(0L, removed); @@ -92,6 +92,7 @@ public void testCleanerScenario1() throws Exception { when(mv1.getCreationMetadata()).thenReturn(mockCM1); MaterializationsInvalidationCache.get().createMaterializedView(mockCM1.getDbName(), mockCM1.getTblName(), mockCM1.getTablesUsed(), mockCM1.getValidTxnList()); + // Format $::::$ Map invalidationInfos = MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( DB_NAME, ImmutableList.of(MV_NAME_1)); @@ -99,11 +100,11 @@ public void testCleanerScenario1() throws Exception { id = 10; BasicTxnInfo txn10 = createTxnInfo(DB_NAME, TBL_NAME_2, id); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, id, id); + DB_NAME, TBL_NAME_2, id, id, false); id = 9; BasicTxnInfo txn9 = createTxnInfo(DB_NAME, TBL_NAME_1, id); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, id, id); + DB_NAME, TBL_NAME_1, id, id, false); // Cleanup (current = 12, duration = 4) -> Removes txn1, txn2, txn3 removed = MaterializationsInvalidationCache.get().cleanup(8L); Assert.assertEquals(0L, removed); @@ -132,15 +133,15 @@ public void testCleanerScenario1() throws Exception { Assert.assertTrue(invalidationInfos.isEmpty()); // Create tbl3 (nothing to do) MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_3, 11, 11); + DB_NAME, TBL_NAME_3, 11, 11, false); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_3, 18, 18); + DB_NAME, TBL_NAME_3, 18, 18, false); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 14, 14); + DB_NAME, TBL_NAME_1, 14, 14, false); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 17, 17); + DB_NAME, TBL_NAME_1, 17, 17, false); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 16, 16); + DB_NAME, TBL_NAME_2, 16, 16, false); // Cleanup (current = 20, duration = 4) -> Removes txn10, txn11 removed = MaterializationsInvalidationCache.get().cleanup(16L); Assert.assertEquals(0L, removed); @@ -149,11 +150,11 @@ public void testCleanerScenario1() throws Exception { DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); Assert.assertTrue(invalidationInfos.isEmpty()); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 12, 12); + DB_NAME, TBL_NAME_1, 12, 12, false); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 15, 15); + DB_NAME, TBL_NAME_2, 15, 15, false); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 7, 7); + DB_NAME, TBL_NAME_2, 7, 7, false); invalidationInfos = MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); @@ -205,16 +206,16 @@ public void testCleanerScenario2() throws Exception { // Cleanup (current = 24, duration = 4) -> Removes txn9, txn14, txn15, txn16, txn17, txn18 // Create tbl1 (nothing to do) MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 1, 1); + DB_NAME, TBL_NAME_1, 1, 1, false); int id = 2; BasicTxnInfo txn2 = createTxnInfo(DB_NAME, TBL_NAME_1, id); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, id, id); + DB_NAME, TBL_NAME_1, id, id, false); // Create tbl2 (nothing to do) id = 3; BasicTxnInfo txn3 = createTxnInfo(DB_NAME, TBL_NAME_1, id); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, id, id); + DB_NAME, TBL_NAME_2, id, id, false); // Cleanup (current = 4, duration = 4) -> Does nothing long removed = MaterializationsInvalidationCache.get().cleanup(0L); Assert.assertEquals(0L, removed); @@ -228,7 +229,8 @@ public void testCleanerScenario2() throws Exception { DB_NAME + "." + TBL_NAME_1, DB_NAME + "." + TBL_NAME_2)); // Create txn list (highWatermark=4;minOpenTxn=Long.MAX_VALUE) - mockCM1.setValidTxnList("3:" + Long.MAX_VALUE + "::"); + mockCM1.setValidTxnList("3$" + DB_NAME + "." + TBL_NAME_1 + ":3:" + Long.MAX_VALUE + "::" + + "$" + DB_NAME + "." + TBL_NAME_2 + ":3:" + Long.MAX_VALUE + "::"); when(mv1.getCreationMetadata()).thenReturn(mockCM1); MaterializationsInvalidationCache.get().createMaterializedView(mockCM1.getDbName(), mockCM1.getTblName(), mockCM1.getTablesUsed(), mockCM1.getValidTxnList()); @@ -239,11 +241,11 @@ public void testCleanerScenario2() throws Exception { id = 10; BasicTxnInfo txn10 = createTxnInfo(DB_NAME, TBL_NAME_2, id); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, id, id); + DB_NAME, TBL_NAME_2, id, id, false); id = 9; BasicTxnInfo txn9 = createTxnInfo(DB_NAME, TBL_NAME_1, id); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, id, id); + DB_NAME, TBL_NAME_1, id, id, false); // Cleanup (current = 12, duration = 4) -> Removes txn1, txn2, txn3 removed = MaterializationsInvalidationCache.get().cleanup(8L); Assert.assertEquals(3L, removed); @@ -261,7 +263,8 @@ public void testCleanerScenario2() throws Exception { DB_NAME + "." + TBL_NAME_1, DB_NAME + "." + TBL_NAME_2)); // Create txn list (highWatermark=10;minOpenTxn=Long.MAX_VALUE) - mockCM2.setValidTxnList("10:" + Long.MAX_VALUE + "::"); + mockCM2.setValidTxnList("10$" + DB_NAME + "." + TBL_NAME_1 + ":10:" + Long.MAX_VALUE + "::" + + "$" + DB_NAME + "." + TBL_NAME_2 + ":10:" + Long.MAX_VALUE + "::"); when(mv2.getCreationMetadata()).thenReturn(mockCM2); MaterializationsInvalidationCache.get().createMaterializedView(mockCM2.getDbName(), mockCM2.getTblName(), mockCM2.getTablesUsed(), mockCM2.getValidTxnList()); @@ -273,15 +276,15 @@ public void testCleanerScenario2() throws Exception { Assert.assertEquals(0L, invalidationInfos.get(MV_NAME_2).getInvalidationTime()); // Create tbl3 (nothing to do) MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_3, 11, 11); + DB_NAME, TBL_NAME_3, 11, 11, false); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_3, 18, 18); + DB_NAME, TBL_NAME_3, 18, 18, false); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 14, 14); + DB_NAME, TBL_NAME_1, 14, 14, false); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 17, 17); + DB_NAME, TBL_NAME_1, 17, 17, false); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 16, 16); + DB_NAME, TBL_NAME_2, 16, 16, false); // Cleanup (current = 20, duration = 4) -> Removes txn10, txn11 removed = MaterializationsInvalidationCache.get().cleanup(16L); Assert.assertEquals(2L, removed); @@ -291,11 +294,11 @@ public void testCleanerScenario2() throws Exception { Assert.assertEquals(9L, invalidationInfos.get(MV_NAME_1).getInvalidationTime()); Assert.assertEquals(14L, invalidationInfos.get(MV_NAME_2).getInvalidationTime()); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 12, 12); + DB_NAME, TBL_NAME_1, 12, 12, false); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 15, 15); + DB_NAME, TBL_NAME_2, 15, 15, false); MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 7, 7); + DB_NAME, TBL_NAME_2, 7, 7, false); invalidationInfos = MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));