diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 4d2eacb498..a13ad28313 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -564,6 +564,7 @@ minillaplocal.query.files=\ materialized_view_create_rewrite_5.q,\ materialized_view_describe.q,\ materialized_view_drop.q,\ + materialized_view_partitioned.q,\ materialized_view_rebuild.q,\ materialized_view_rewrite_empty.q,\ materialized_view_rewrite_1.q,\ @@ -575,6 +576,8 @@ minillaplocal.query.files=\ materialized_view_rewrite_7.q,\ materialized_view_rewrite_8.q,\ materialized_view_rewrite_9.q,\ + materialized_view_rewrite_part_1.q,\ + materialized_view_rewrite_part_2.q,\ materialized_view_rewrite_ssb.q,\ materialized_view_rewrite_ssb_2.q,\ mapjoin_decimal.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index 5c9162f1e1..696227be48 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -96,9 +96,6 @@ private final ConcurrentMap> materializedViews = new ConcurrentHashMap>(); - /* If this boolean is true, we bypass the cache. */ - private boolean dummy; - /* Whether the cache has been initialized or not. */ private AtomicBoolean initialized = new AtomicBoolean(false); @@ -137,7 +134,7 @@ public void init() { } public void init(Hive db) { - dummy = db.getConf().get(HiveConf.ConfVars.HIVE_SERVER2_MATERIALIZED_VIEWS_REGISTRY_IMPL.varname) + final boolean dummy = db.getConf().get(HiveConf.ConfVars.HIVE_SERVER2_MATERIALIZED_VIEWS_REGISTRY_IMPL.varname) .equals("DUMMY"); if (dummy) { // Dummy registry does not cache information and forwards all requests to metastore @@ -162,9 +159,11 @@ private Loader(Hive db) { public void run() { try { SessionState.start(db.getConf()); + final boolean cache = !db.getConf() + .get(HiveConf.ConfVars.HIVE_SERVER2_MATERIALIZED_VIEWS_REGISTRY_IMPL.varname).equals("DUMMY"); for (String dbName : db.getAllDatabases()) { for (Table mv : db.getAllMaterializedViewObjects(dbName)) { - addMaterializedView(db.getConf(), mv, OpType.LOAD); + addMaterializedView(db.getConf(), mv, OpType.LOAD, cache); } } initialized.set(true); @@ -185,7 +184,9 @@ public boolean isInitialized() { * @param materializedViewTable the materialized view */ public RelOptMaterialization createMaterializedView(HiveConf conf, Table materializedViewTable) { - return addMaterializedView(conf, materializedViewTable, OpType.CREATE); + final boolean cache = !conf.get(HiveConf.ConfVars.HIVE_SERVER2_MATERIALIZED_VIEWS_REGISTRY_IMPL.varname) + .equals("DUMMY"); + return addMaterializedView(conf, materializedViewTable, OpType.CREATE, cache); } /** @@ -193,7 +194,8 @@ public RelOptMaterialization createMaterializedView(HiveConf conf, Table materia * * @param materializedViewTable the materialized view */ - private RelOptMaterialization addMaterializedView(HiveConf conf, Table materializedViewTable, OpType opType) { + private RelOptMaterialization addMaterializedView(HiveConf conf, Table materializedViewTable, + OpType opType, boolean cache) { // Bail out if it is not enabled for rewriting if (!materializedViewTable.isRewriteEnabled()) { LOG.debug("Materialized view " + materializedViewTable.getCompleteName() + @@ -204,7 +206,7 @@ private RelOptMaterialization addMaterializedView(HiveConf conf, Table materiali // We are going to create the map for each view in the given database ConcurrentMap cq = new ConcurrentHashMap(); - if (!dummy) { + if (cache) { // If we are caching the MV, we include it in the cache final ConcurrentMap prevCq = materializedViews.putIfAbsent( materializedViewTable.getDbName(), cq); @@ -219,13 +221,13 @@ private RelOptMaterialization addMaterializedView(HiveConf conf, Table materiali final RelNode viewScan = createMaterializedViewScan(conf, materializedViewTable); if (viewScan == null) { LOG.warn("Materialized view " + materializedViewTable.getCompleteName() + - " ignored; error creating view replacement"); + " ignored; error creating view replacement"); return null; } final RelNode queryRel = parseQuery(conf, viewQuery); if (queryRel == null) { LOG.warn("Materialized view " + materializedViewTable.getCompleteName() + - " ignored; error parsing original query"); + " ignored; error parsing original query"); return null; } @@ -261,10 +263,6 @@ public void dropMaterializedView(Table materializedViewTable) { * @param tableName the name for the materialized view to remove */ public void dropMaterializedView(String dbName, String tableName) { - if (dummy) { - // Nothing to do - return; - } ConcurrentMap dbMap = materializedViews.get(dbName); if (dbMap != null) { dbMap.remove(tableName); @@ -288,8 +286,8 @@ private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable // 0. Recreate cluster final RelOptPlanner planner = CalcitePlanner.createPlanner(conf); final RexBuilder rexBuilder = new RexBuilder( - new JavaTypeFactoryImpl( - new HiveTypeSystemImpl())); + new JavaTypeFactoryImpl( + new HiveTypeSystemImpl())); final RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder); // 1. Create column schema @@ -380,7 +378,7 @@ private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable List intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL); rowType = dtFactory.createStructType(druidColTypes, druidColNames); - RelOptHiveTable optTable = new RelOptHiveTable(null, fullyQualifiedTabName, + RelOptHiveTable optTable = new RelOptHiveTable(null, cluster.getTypeFactory(), fullyQualifiedTabName, rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<>(), conf, new HashMap<>(), new HashMap<>(), new AtomicInteger()); DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false), @@ -392,7 +390,7 @@ private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable optTable, druidTable, ImmutableList.of(scan), ImmutableMap.of()); } else { // Build Hive Table Scan Rel - RelOptHiveTable optTable = new RelOptHiveTable(null, fullyQualifiedTabName, + RelOptHiveTable optTable = new RelOptHiveTable(null, cluster.getTypeFactory(), fullyQualifiedTabName, rowType, viewTable, nonPartitionColumns, partitionColumns, new ArrayList<>(), conf, new HashMap<>(), new HashMap<>(), new AtomicInteger()); tableRel = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), optTable, @@ -448,4 +446,4 @@ private static TableType obtainTableType(Table tabMetaData) { LOAD // already created view being loaded } -} +} \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java index 943227fb19..37e6d4c850 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java @@ -44,6 +44,7 @@ import org.apache.calcite.rel.RelReferentialConstraintImpl; import org.apache.calcite.rel.logical.LogicalTableScan; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexNode; import org.apache.calcite.schema.ColumnStrategy; @@ -87,6 +88,7 @@ //~ Instance fields -------------------------------------------------------- private final RelOptSchema schema; + private final RelDataTypeFactory typeFactory; private final RelDataType rowType; private final List qualifiedTblName; private final String name; @@ -110,12 +112,13 @@ protected static final Logger LOG = LoggerFactory.getLogger(RelOptHiveTable.class.getName()); - public RelOptHiveTable(RelOptSchema calciteSchema, List qualifiedTblName, + public RelOptHiveTable(RelOptSchema calciteSchema, RelDataTypeFactory typeFactory, List qualifiedTblName, RelDataType rowType, Table hiveTblMetadata, List hiveNonPartitionCols, List hivePartitionCols, List hiveVirtualCols, HiveConf hconf, Map partitionCache, Map colStatsCache, AtomicInteger noColsMissingStats) { this.schema = calciteSchema; + this.typeFactory = typeFactory; this.qualifiedTblName = ImmutableList.copyOf(qualifiedTblName); this.name = this.qualifiedTblName.stream().collect(Collectors.joining(".")); this.rowType = rowType; @@ -155,6 +158,10 @@ public RelOptSchema getRelOptSchema() { return schema; } + public RelDataTypeFactory getTypeFactory() { + return typeFactory; + } + @Override public Expression getExpression(Class clazz) { throw new UnsupportedOperationException(); @@ -201,7 +208,7 @@ public RelOptHiveTable copy(RelDataType newRowType) { } // 3. Build new Table - return new RelOptHiveTable(this.schema, this.qualifiedTblName, newRowType, + return new RelOptHiveTable(this.schema, this.typeFactory, this.qualifiedTblName, newRowType, this.hiveTblMetadata, newHiveNonPartitionCols, newHivePartitionCols, newHiveVirtualCols, this.hiveConf, this.partitionCache, this.colStatsCache, this.noColsMissingStats); } @@ -439,7 +446,7 @@ public void computePartitionList(HiveConf conf, RexNode pruneNode, Set // We have valid pruning expressions, only retrieve qualifying partitions ExprNodeDesc pruneExpr = pruneNode.accept(new ExprNodeConverter(getName(), getRowType(), - partOrVirtualCols, this.getRelOptSchema().getTypeFactory())); + partOrVirtualCols, getTypeFactory())); partitionList = PartitionPruner.prune(hiveTblMetadata, pruneExpr, conf, getName(), partitionCache); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 61396e76ab..e6e033066d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -2152,7 +2152,6 @@ private RelNode applyMaterializedViewRewriting(RelOptPlanner planner, RelNode ba final RelOptCluster optCluster = basePlan.getCluster(); final PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); final RelNode calcitePreMVRewritingPlan = basePlan; // Add views to planner @@ -2216,6 +2215,8 @@ private RelNode copyNodeScan(RelNode scan) { LOG.warn("Exception loading materialized views", e); } if (!materializations.isEmpty()) { + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + // Use Calcite cost model for view rewriting optCluster.invalidateMetadataQuery(); RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(DefaultRelMetadataProvider.INSTANCE)); @@ -2240,7 +2241,9 @@ private RelNode copyNodeScan(RelNode scan) { // Restore default cost model optCluster.invalidateMetadataQuery(); RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(mdProvider)); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: View-based rewriting"); + if (calcitePreMVRewritingPlan != basePlan) { // A rewriting was produced, we will check whether it was part of an incremental rebuild // to try to replace INSERT OVERWRITE by INSERT @@ -2877,7 +2880,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false), dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, intervals, null, null); - RelOptHiveTable optTable = new RelOptHiveTable(relOptSchema, fullyQualifiedTabName, + RelOptHiveTable optTable = new RelOptHiveTable(relOptSchema, relOptSchema.getTypeFactory(), fullyQualifiedTabName, rowType, tabMetaData, nonPartitionColumns, partitionColumns, virtualCols, conf, partitionCache, colStatsCache, noColsMissingStats); final TableScan scan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), @@ -2888,7 +2891,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc tableRel = DruidQuery.create(cluster, cluster.traitSetOf(BindableConvention.INSTANCE), optTable, druidTable, ImmutableList.of(scan), DruidSqlOperatorConverter.getDefaultMap()); } else if (tableType == TableType.JDBC) { - RelOptHiveTable optTable = new RelOptHiveTable(relOptSchema, fullyQualifiedTabName, + RelOptHiveTable optTable = new RelOptHiveTable(relOptSchema, relOptSchema.getTypeFactory(), fullyQualifiedTabName, rowType, tabMetaData, nonPartitionColumns, partitionColumns, virtualCols, conf, partitionCache, colStatsCache, noColsMissingStats); @@ -2929,7 +2932,7 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc fullyQualifiedTabName.add(tabMetaData.getDbName()); } fullyQualifiedTabName.add(tabMetaData.getTableName()); - RelOptHiveTable optTable = new RelOptHiveTable(relOptSchema, fullyQualifiedTabName, + RelOptHiveTable optTable = new RelOptHiveTable(relOptSchema, relOptSchema.getTypeFactory(), fullyQualifiedTabName, rowType, tabMetaData, nonPartitionColumns, partitionColumns, virtualCols, conf, partitionCache, colStatsCache, noColsMissingStats); // Build Hive Table Scan Rel diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 49f5487f40..d581c602c3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -1928,6 +1928,19 @@ createViewStatement ) ; +viewPartition +@init { pushMsg("view partition specification", state); } +@after { popMsg(state); } + : KW_PARTITIONED KW_ON LPAREN columnNameList RPAREN + -> ^(TOK_VIEWPARTCOLS columnNameList) + ; + +dropViewStatement +@init { pushMsg("drop view statement", state); } +@after { popMsg(state); } + : KW_DROP KW_VIEW ifExists? viewName -> ^(TOK_DROPVIEW viewName ifExists?) + ; + createMaterializedViewStatement @init { pushMsg("create materialized view statement", state); @@ -1935,7 +1948,7 @@ createMaterializedViewStatement @after { popMsg(state); } : KW_CREATE KW_MATERIALIZED KW_VIEW (ifNotExists)? name=tableName rewriteDisabled? tableComment? tableRowFormat? tableFileFormat? tableLocation? - tablePropertiesPrefixed? KW_AS selectStatementWithCTE + viewPartition? tablePropertiesPrefixed? KW_AS selectStatementWithCTE -> ^(TOK_CREATE_MATERIALIZED_VIEW $name ifNotExists? rewriteDisabled? @@ -1943,24 +1956,12 @@ createMaterializedViewStatement tableRowFormat? tableFileFormat? tableLocation? + viewPartition? tablePropertiesPrefixed? selectStatementWithCTE ) ; -viewPartition -@init { pushMsg("view partition specification", state); } -@after { popMsg(state); } - : KW_PARTITIONED KW_ON LPAREN columnNameList RPAREN - -> ^(TOK_VIEWPARTCOLS columnNameList) - ; - -dropViewStatement -@init { pushMsg("drop view statement", state); } -@after { popMsg(state); } - : KW_DROP KW_VIEW ifExists? viewName -> ^(TOK_DROPVIEW viewName ifExists?) - ; - dropMaterializedViewStatement @init { pushMsg("drop materialized view statement", state); } @after { popMsg(state); } diff --git a/ql/src/test/queries/clientpositive/materialized_view_partitioned.q b/ql/src/test/queries/clientpositive/materialized_view_partitioned.q new file mode 100644 index 0000000000..fcc7c1f0c9 --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_partitioned.q @@ -0,0 +1,62 @@ +--! qt:dataset:src + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src; + +EXPLAIN +CREATE MATERIALIZED VIEW partition_mv_1 PARTITIONED ON (key) AS +SELECT value, key FROM src_txn where key > 200 and key < 250; + +CREATE MATERIALIZED VIEW partition_mv_1 PARTITIONED ON (key) AS +SELECT value, key FROM src_txn where key > 200 and key < 250; + +DESCRIBE FORMATTED partition_mv_1; + +EXPLAIN +SELECT * FROM partition_mv_1 where key = 238; + +SELECT * FROM partition_mv_1 where key = 238; + +CREATE MATERIALIZED VIEW partition_mv_2 PARTITIONED ON (value) AS +SELECT key, value FROM src_txn where key > 200 and key < 250; + +EXPLAIN +SELECT * FROM partition_mv_2 where value = 'val_238'; + +SELECT * FROM partition_mv_2 where value = 'val_238'; + +EXPLAIN +SELECT value FROM partition_mv_2 where key = 238; + +SELECT value FROM partition_mv_2 where key = 238; + +INSERT INTO src_txn VALUES (238, 'val_238_n'); + +EXPLAIN +ALTER MATERIALIZED VIEW partition_mv_1 REBUILD; + +ALTER MATERIALIZED VIEW partition_mv_1 REBUILD; + +SELECT * FROM partition_mv_1 where key = 238; + +SELECT * FROM partition_mv_2 where key = 238; + +CREATE TABLE src_txn_2 stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src; + +CREATE MATERIALIZED VIEW partition_mv_3 PARTITIONED ON (key) AS +SELECT src_txn.value, src_txn.key FROM src_txn, src_txn_2 +WHERE src_txn.key = src_txn_2.key + AND src_txn.key > 200 AND src_txn.key < 250; + +INSERT INTO src_txn VALUES (238, 'val_238_n2'); + +EXPLAIN +ALTER MATERIALIZED VIEW partition_mv_3 REBUILD; + +ALTER MATERIALIZED VIEW partition_mv_3 REBUILD; + +SELECT * FROM partition_mv_3 where key = 238; diff --git a/ql/src/test/queries/clientpositive/materialized_view_rewrite_part_1.q b/ql/src/test/queries/clientpositive/materialized_view_rewrite_part_1.q new file mode 100644 index 0000000000..5a2e74c8a0 --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_rewrite_part_1.q @@ -0,0 +1,145 @@ +-- SORT_QUERY_RESULTS + +set hive.vectorized.execution.enabled=false; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.stats.fetch.column.stats=true; +set hive.materializedview.rewriting=true; + +create table emps_n30 ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into emps_n30 values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250); +analyze table emps_n30 compute statistics for columns; + +create table depts_n20 ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into depts_n20 values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20); +analyze table depts_n20 compute statistics for columns; + +create table dependents_n20 ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into dependents_n20 values (10, 'Michael'), (10, 'Jane'); +analyze table dependents_n20 compute statistics for columns; + +create table locations_n20 ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into locations_n20 values (10, 'San Francisco'), (10, 'San Diego'); +analyze table locations_n20 compute statistics for columns; + +alter table emps_n30 add constraint pk1 primary key (empid) disable novalidate rely; +alter table depts_n20 add constraint pk2 primary key (deptno) disable novalidate rely; +alter table dependents_n20 add constraint pk3 primary key (empid) disable novalidate rely; +alter table locations_n20 add constraint pk4 primary key (locationid) disable novalidate rely; + +alter table emps_n30 add constraint fk1 foreign key (deptno) references depts_n20(deptno) disable novalidate rely; +alter table depts_n20 add constraint fk2 foreign key (locationid) references locations_n20(locationid) disable novalidate rely; + +-- EXAMPLE 1 +create materialized view mv1_part_n2 partitioned on (deptno) as +select * from emps_n30 where empid < 150; +analyze table mv1_part_n2 compute statistics for columns; + +explain +select * +from (select * from emps_n30 where empid < 120) t +join depts_n20 using (deptno); + +select * +from (select * from emps_n30 where empid < 120) t +join depts_n20 using (deptno); + +drop materialized view mv1_part_n2; + +-- EXAMPLE 2 +create materialized view mv1_part_n2 partitioned on (deptno) as +select deptno, name, salary, commission +from emps_n30; +analyze table mv1_part_n2 compute statistics for columns; + +explain +select emps_n30.name, emps_n30.salary, emps_n30.commission +from emps_n30 +join depts_n20 using (deptno); + +select emps_n30.name, emps_n30.salary, emps_n30.commission +from emps_n30 +join depts_n20 using (deptno); + +drop materialized view mv1_part_n2; + +-- EXAMPLE 4 +create materialized view mv1_part_n2 partitioned on (deptno) as +select * from emps_n30 where empid < 200; +analyze table mv1_part_n2 compute statistics for columns; + +explain +select * from emps_n30 where empid > 120 +union all select * from emps_n30 where empid < 150; + +select * from emps_n30 where empid > 120 +union all select * from emps_n30 where empid < 150; + +drop materialized view mv1_part_n2; + +-- EXAMPLE 5 +create materialized view mv1_part_n2 partitioned on (name) as +select name, salary from emps_n30 group by name, salary; +analyze table mv1_part_n2 compute statistics for columns; + +explain +select name, salary from emps_n30 group by name, salary; + +select name, salary from emps_n30 group by name, salary; + +drop materialized view mv1_part_n2; + +-- EXAMPLE 6 +create materialized view mv1_part_n2 partitioned on (name) as +select name, salary from emps_n30 group by name, salary; +analyze table mv1_part_n2 compute statistics for columns; + +explain +select name from emps_n30 group by name; + +select name from emps_n30 group by name; + +drop materialized view mv1_part_n2; + +-- EXAMPLE 7 +create materialized view mv1_part_n2 partitioned on (name) as +select name, salary from emps_n30 where deptno = 10 group by name, salary; +analyze table mv1_part_n2 compute statistics for columns; + +explain +select name from emps_n30 where deptno = 10 group by name; + +select name from emps_n30 where deptno = 10 group by name; + +drop materialized view mv1_part_n2; + +-- EXAMPLE 9 +create materialized view mv1_part_n2 partitioned on (name) as +select name, salary, count(*) as c, sum(empid) as s +from emps_n30 group by name, salary; +analyze table mv1_part_n2 compute statistics for columns; + +explain +select name from emps_n30 group by name; + +select name from emps_n30 group by name; + +drop materialized view mv1_part_n2; diff --git a/ql/src/test/queries/clientpositive/materialized_view_rewrite_part_2.q b/ql/src/test/queries/clientpositive/materialized_view_rewrite_part_2.q new file mode 100644 index 0000000000..505f7507bc --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_rewrite_part_2.q @@ -0,0 +1,164 @@ +-- SORT_QUERY_RESULTS + +set hive.vectorized.execution.enabled=false; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.stats.fetch.column.stats=true; +set hive.materializedview.rewriting=true; + +create table emps_n00 ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into emps_n00 values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250); +analyze table emps_n00 compute statistics for columns; + +create table depts_n00 ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into depts_n00 values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20); +analyze table depts_n00 compute statistics for columns; + +create table dependents_n00 ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into dependents_n00 values (10, 'Michael'), (10, 'Jane'); +analyze table dependents_n00 compute statistics for columns; + +create table locations_n00 ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true'); +insert into locations_n00 values (10, 'San Francisco'), (10, 'San Diego'); +analyze table locations_n00 compute statistics for columns; + +alter table emps_n00 add constraint pk1 primary key (empid) disable novalidate rely; +alter table depts_n00 add constraint pk2 primary key (deptno) disable novalidate rely; +alter table dependents_n00 add constraint pk3 primary key (empid) disable novalidate rely; +alter table locations_n00 add constraint pk4 primary key (locationid) disable novalidate rely; + +alter table emps_n00 add constraint fk1 foreign key (deptno) references depts_n00(deptno) disable novalidate rely; +alter table depts_n00 add constraint fk2 foreign key (locationid) references locations_n00(locationid) disable novalidate rely; + +-- EXAMPLE 16 +create materialized view mv1_part_n0 partitioned on (deptno) as +select empid, depts_n00.deptno as deptno from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 10 +group by empid, depts_n00.deptno; +analyze table mv1_part_n0 compute statistics for columns; + +explain +select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno; + +select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno; + +drop materialized view mv1_part_n0; + +-- EXAMPLE 17 +create materialized view mv1_part_n0 partitioned on (deptno) as +select depts_n00.deptno as deptno, empid from depts_n00 +join emps_n00 using (deptno) where depts_n00.deptno > 10 +group by empid, depts_n00.deptno; +analyze table mv1_part_n0 compute statistics for columns; + +explain +select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno; + +select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno; + +drop materialized view mv1_part_n0; + +-- EXAMPLE 18 +create materialized view mv1_part_n0 partitioned on (deptno) as +select empid, depts_n00.deptno as deptno from emps_n00 +join depts_n00 using (deptno) where emps_n00.deptno > 10 +group by empid, depts_n00.deptno; +analyze table mv1_part_n0 compute statistics for columns; + +explain +select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno; + +select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno; + +drop materialized view mv1_part_n0; + +-- EXAMPLE 19 +create materialized view mv1_part_n0 partitioned on (deptno) as +select depts_n00.deptno as deptno, emps_n00.empid from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 10 +group by depts_n00.deptno, emps_n00.empid; +analyze table mv1_part_n0 compute statistics for columns; + +explain +select depts_n00.deptno from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 15 +group by depts_n00.deptno, emps_n00.empid; + +select depts_n00.deptno from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 15 +group by depts_n00.deptno, emps_n00.empid; + +drop materialized view mv1_part_n0; + +-- EXAMPLE 20 +create materialized view mv1_part_n0 partitioned on (deptno) as +select depts_n00.deptno as deptno, emps_n00.empid from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 10 +group by depts_n00.deptno, emps_n00.empid; +analyze table mv1_part_n0 compute statistics for columns; + +explain +select depts_n00.deptno from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 15 +group by depts_n00.deptno; + +select depts_n00.deptno from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 15 +group by depts_n00.deptno; + +drop materialized view mv1_part_n0; + +-- EXAMPLE 23 +create materialized view mv1_part_n0 partitioned on (deptno2) as +select depts_n00.name, dependents_n00.name as name2, emps_n00.deptno, depts_n00.deptno as deptno2, dependents_n00.empid +from depts_n00, dependents_n00, emps_n00 +where depts_n00.deptno > 10 +group by depts_n00.name, dependents_n00.name, emps_n00.deptno, depts_n00.deptno, dependents_n00.empid; +analyze table mv1_part_n0 compute statistics for columns; + +explain +select dependents_n00.empid +from depts_n00 +join dependents_n00 on (depts_n00.name = dependents_n00.name) +join emps_n00 on (emps_n00.deptno = depts_n00.deptno) +where depts_n00.deptno > 10 +group by dependents_n00.empid; + +select dependents_n00.empid +from depts_n00 +join dependents_n00 on (depts_n00.name = dependents_n00.name) +join emps_n00 on (emps_n00.deptno = depts_n00.deptno) +where depts_n00.deptno > 10 +group by dependents_n00.empid; + +drop materialized view mv1_part_n0; diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out new file mode 100644 index 0000000000..3852945776 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out @@ -0,0 +1,1075 @@ +PREHOOK: query: CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src_txn +POSTHOOK: query: CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_txn.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW partition_mv_1 PARTITIONED ON (key) AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW partition_mv_1 PARTITIONED ON (key) AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-4 + Stage-3 depends on stages: Stage-0, Stage-4 + Stage-5 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) < 250.0D) and (UDFToDouble(key) > 200.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 19268 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 19268 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 55 Data size: 19268 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.partition_mv_1 + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Create View Operator: + Create View + partition columns: key string + columns: value string + expanded text: SELECT `src_txn`.`value`, `src_txn`.`key` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250 + name: default.partition_mv_1 + original text: SELECT value, key FROM src_txn where key > 200 and key < 250 + rewrite enabled: true + + Stage: Stage-0 + Move Operator + tables: + partition: + key + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.partition_mv_1 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + + Stage: Stage-5 + Materialized View Work + +PREHOOK: query: CREATE MATERIALIZED VIEW partition_mv_1 PARTITIONED ON (key) AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@partition_mv_1 +PREHOOK: Output: default@partition_mv_1 +POSTHOOK: query: CREATE MATERIALIZED VIEW partition_mv_1 PARTITIONED ON (key) AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@partition_mv_1 +POSTHOOK: Output: default@partition_mv_1@key=201 +POSTHOOK: Output: default@partition_mv_1@key=202 +POSTHOOK: Output: default@partition_mv_1@key=203 +POSTHOOK: Output: default@partition_mv_1@key=205 +POSTHOOK: Output: default@partition_mv_1@key=207 +POSTHOOK: Output: default@partition_mv_1@key=208 +POSTHOOK: Output: default@partition_mv_1@key=209 +POSTHOOK: Output: default@partition_mv_1@key=213 +POSTHOOK: Output: default@partition_mv_1@key=214 +POSTHOOK: Output: default@partition_mv_1@key=216 +POSTHOOK: Output: default@partition_mv_1@key=217 +POSTHOOK: Output: default@partition_mv_1@key=218 +POSTHOOK: Output: default@partition_mv_1@key=219 +POSTHOOK: Output: default@partition_mv_1@key=221 +POSTHOOK: Output: default@partition_mv_1@key=222 +POSTHOOK: Output: default@partition_mv_1@key=223 +POSTHOOK: Output: default@partition_mv_1@key=224 +POSTHOOK: Output: default@partition_mv_1@key=226 +POSTHOOK: Output: default@partition_mv_1@key=228 +POSTHOOK: Output: default@partition_mv_1@key=229 +POSTHOOK: Output: default@partition_mv_1@key=230 +POSTHOOK: Output: default@partition_mv_1@key=233 +POSTHOOK: Output: default@partition_mv_1@key=235 +POSTHOOK: Output: default@partition_mv_1@key=237 +POSTHOOK: Output: default@partition_mv_1@key=238 +POSTHOOK: Output: default@partition_mv_1@key=239 +POSTHOOK: Output: default@partition_mv_1@key=241 +POSTHOOK: Output: default@partition_mv_1@key=242 +POSTHOOK: Output: default@partition_mv_1@key=244 +POSTHOOK: Output: default@partition_mv_1@key=247 +POSTHOOK: Output: default@partition_mv_1@key=248 +POSTHOOK: Output: default@partition_mv_1@key=249 +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=201).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=202).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=203).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=205).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=207).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=208).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=209).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=213).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=214).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=216).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=217).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=218).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=219).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=221).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=222).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=223).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=224).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=226).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=228).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=229).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=230).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=233).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=235).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=237).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=238).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=239).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=241).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=242).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=244).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=247).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=248).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=249).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: DESCRIBE FORMATTED partition_mv_1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@partition_mv_1 +POSTHOOK: query: DESCRIBE FORMATTED partition_mv_1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@partition_mv_1 +# col_name data_type comment +value string + +# Partition Information +# col_name data_type comment +key string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + numFiles 32 + numPartitions 32 + numRows 55 + rawDataSize 5005 + totalSize 7618 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: SELECT value, key FROM src_txn where key > 200 and key < 250 +View Expanded Text: SELECT `src_txn`.`value`, `src_txn`.`key` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250 +View Rewrite Enabled: Yes +PREHOOK: query: EXPLAIN +SELECT * FROM partition_mv_1 where key = 238 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT * FROM partition_mv_1 where key = 238 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: partition_mv_1 + filterExpr: (238.0D = 238.0D) (type: boolean) + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT * FROM partition_mv_1 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_mv_1 +PREHOOK: Input: default@partition_mv_1@key=238 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM partition_mv_1 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_mv_1 +POSTHOOK: Input: default@partition_mv_1@key=238 +#### A masked pattern was here #### +val_238 238 +val_238 238 +PREHOOK: query: CREATE MATERIALIZED VIEW partition_mv_2 PARTITIONED ON (value) AS +SELECT key, value FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@partition_mv_2 +PREHOOK: Output: default@partition_mv_2 +POSTHOOK: query: CREATE MATERIALIZED VIEW partition_mv_2 PARTITIONED ON (value) AS +SELECT key, value FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@partition_mv_2 +POSTHOOK: Output: default@partition_mv_2@value=val_201 +POSTHOOK: Output: default@partition_mv_2@value=val_202 +POSTHOOK: Output: default@partition_mv_2@value=val_203 +POSTHOOK: Output: default@partition_mv_2@value=val_205 +POSTHOOK: Output: default@partition_mv_2@value=val_207 +POSTHOOK: Output: default@partition_mv_2@value=val_208 +POSTHOOK: Output: default@partition_mv_2@value=val_209 +POSTHOOK: Output: default@partition_mv_2@value=val_213 +POSTHOOK: Output: default@partition_mv_2@value=val_214 +POSTHOOK: Output: default@partition_mv_2@value=val_216 +POSTHOOK: Output: default@partition_mv_2@value=val_217 +POSTHOOK: Output: default@partition_mv_2@value=val_218 +POSTHOOK: Output: default@partition_mv_2@value=val_219 +POSTHOOK: Output: default@partition_mv_2@value=val_221 +POSTHOOK: Output: default@partition_mv_2@value=val_222 +POSTHOOK: Output: default@partition_mv_2@value=val_223 +POSTHOOK: Output: default@partition_mv_2@value=val_224 +POSTHOOK: Output: default@partition_mv_2@value=val_226 +POSTHOOK: Output: default@partition_mv_2@value=val_228 +POSTHOOK: Output: default@partition_mv_2@value=val_229 +POSTHOOK: Output: default@partition_mv_2@value=val_230 +POSTHOOK: Output: default@partition_mv_2@value=val_233 +POSTHOOK: Output: default@partition_mv_2@value=val_235 +POSTHOOK: Output: default@partition_mv_2@value=val_237 +POSTHOOK: Output: default@partition_mv_2@value=val_238 +POSTHOOK: Output: default@partition_mv_2@value=val_239 +POSTHOOK: Output: default@partition_mv_2@value=val_241 +POSTHOOK: Output: default@partition_mv_2@value=val_242 +POSTHOOK: Output: default@partition_mv_2@value=val_244 +POSTHOOK: Output: default@partition_mv_2@value=val_247 +POSTHOOK: Output: default@partition_mv_2@value=val_248 +POSTHOOK: Output: default@partition_mv_2@value=val_249 +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_201).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_202).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_203).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_205).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_207).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_208).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_209).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_213).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_214).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_216).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_217).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_218).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_219).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_221).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_222).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_223).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_224).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_226).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_228).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_229).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_230).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_233).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_235).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_237).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_238).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_239).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_241).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_242).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_244).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_247).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_248).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_2 PARTITION(value=val_249).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +PREHOOK: query: EXPLAIN +SELECT * FROM partition_mv_2 where value = 'val_238' +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT * FROM partition_mv_2 where value = 'val_238' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: partition_mv_2 + filterExpr: (value = 'val_238') (type: boolean) + Select Operator + expressions: key (type: string), 'val_238' (type: string) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT * FROM partition_mv_2 where value = 'val_238' +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_mv_2 +PREHOOK: Input: default@partition_mv_2@value=val_238 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM partition_mv_2 where value = 'val_238' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_mv_2 +POSTHOOK: Input: default@partition_mv_2@value=val_238 +#### A masked pattern was here #### +238 val_238 +238 val_238 +PREHOOK: query: EXPLAIN +SELECT value FROM partition_mv_2 where key = 238 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT value FROM partition_mv_2 where key = 238 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: partition_mv_2 + filterExpr: (UDFToDouble(key) = 238.0D) (type: boolean) + Filter Operator + predicate: (UDFToDouble(key) = 238.0D) (type: boolean) + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT value FROM partition_mv_2 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_mv_2 +PREHOOK: Input: default@partition_mv_2@value=val_201 +PREHOOK: Input: default@partition_mv_2@value=val_202 +PREHOOK: Input: default@partition_mv_2@value=val_203 +PREHOOK: Input: default@partition_mv_2@value=val_205 +PREHOOK: Input: default@partition_mv_2@value=val_207 +PREHOOK: Input: default@partition_mv_2@value=val_208 +PREHOOK: Input: default@partition_mv_2@value=val_209 +PREHOOK: Input: default@partition_mv_2@value=val_213 +PREHOOK: Input: default@partition_mv_2@value=val_214 +PREHOOK: Input: default@partition_mv_2@value=val_216 +PREHOOK: Input: default@partition_mv_2@value=val_217 +PREHOOK: Input: default@partition_mv_2@value=val_218 +PREHOOK: Input: default@partition_mv_2@value=val_219 +PREHOOK: Input: default@partition_mv_2@value=val_221 +PREHOOK: Input: default@partition_mv_2@value=val_222 +PREHOOK: Input: default@partition_mv_2@value=val_223 +PREHOOK: Input: default@partition_mv_2@value=val_224 +PREHOOK: Input: default@partition_mv_2@value=val_226 +PREHOOK: Input: default@partition_mv_2@value=val_228 +PREHOOK: Input: default@partition_mv_2@value=val_229 +PREHOOK: Input: default@partition_mv_2@value=val_230 +PREHOOK: Input: default@partition_mv_2@value=val_233 +PREHOOK: Input: default@partition_mv_2@value=val_235 +PREHOOK: Input: default@partition_mv_2@value=val_237 +PREHOOK: Input: default@partition_mv_2@value=val_238 +PREHOOK: Input: default@partition_mv_2@value=val_239 +PREHOOK: Input: default@partition_mv_2@value=val_241 +PREHOOK: Input: default@partition_mv_2@value=val_242 +PREHOOK: Input: default@partition_mv_2@value=val_244 +PREHOOK: Input: default@partition_mv_2@value=val_247 +PREHOOK: Input: default@partition_mv_2@value=val_248 +PREHOOK: Input: default@partition_mv_2@value=val_249 +#### A masked pattern was here #### +POSTHOOK: query: SELECT value FROM partition_mv_2 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_mv_2 +POSTHOOK: Input: default@partition_mv_2@value=val_201 +POSTHOOK: Input: default@partition_mv_2@value=val_202 +POSTHOOK: Input: default@partition_mv_2@value=val_203 +POSTHOOK: Input: default@partition_mv_2@value=val_205 +POSTHOOK: Input: default@partition_mv_2@value=val_207 +POSTHOOK: Input: default@partition_mv_2@value=val_208 +POSTHOOK: Input: default@partition_mv_2@value=val_209 +POSTHOOK: Input: default@partition_mv_2@value=val_213 +POSTHOOK: Input: default@partition_mv_2@value=val_214 +POSTHOOK: Input: default@partition_mv_2@value=val_216 +POSTHOOK: Input: default@partition_mv_2@value=val_217 +POSTHOOK: Input: default@partition_mv_2@value=val_218 +POSTHOOK: Input: default@partition_mv_2@value=val_219 +POSTHOOK: Input: default@partition_mv_2@value=val_221 +POSTHOOK: Input: default@partition_mv_2@value=val_222 +POSTHOOK: Input: default@partition_mv_2@value=val_223 +POSTHOOK: Input: default@partition_mv_2@value=val_224 +POSTHOOK: Input: default@partition_mv_2@value=val_226 +POSTHOOK: Input: default@partition_mv_2@value=val_228 +POSTHOOK: Input: default@partition_mv_2@value=val_229 +POSTHOOK: Input: default@partition_mv_2@value=val_230 +POSTHOOK: Input: default@partition_mv_2@value=val_233 +POSTHOOK: Input: default@partition_mv_2@value=val_235 +POSTHOOK: Input: default@partition_mv_2@value=val_237 +POSTHOOK: Input: default@partition_mv_2@value=val_238 +POSTHOOK: Input: default@partition_mv_2@value=val_239 +POSTHOOK: Input: default@partition_mv_2@value=val_241 +POSTHOOK: Input: default@partition_mv_2@value=val_242 +POSTHOOK: Input: default@partition_mv_2@value=val_244 +POSTHOOK: Input: default@partition_mv_2@value=val_247 +POSTHOOK: Input: default@partition_mv_2@value=val_248 +POSTHOOK: Input: default@partition_mv_2@value=val_249 +#### A masked pattern was here #### +val_238 +val_238 +PREHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@src_txn +POSTHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SCRIPT [] +POSTHOOK: Lineage: src_txn.value SCRIPT [] +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW partition_mv_1 REBUILD +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW partition_mv_1 REBUILD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 501 Data size: 175536 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) < 250.0D) and (UDFToDouble(key) > 200.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 19270 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 19270 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 55 Data size: 19270 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.partition_mv_1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: value, key + Statistics: Num rows: 55 Data size: 19270 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(value, 'hll') + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 19270 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 19270 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: struct) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 27 Data size: 9459 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: struct), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 27 Data size: 9459 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 9459 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + key + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.partition_mv_1 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value + Column Types: string + Table: default.partition_mv_1 + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW partition_mv_1 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Output: default@partition_mv_1 +POSTHOOK: query: ALTER MATERIALIZED VIEW partition_mv_1 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: default@partition_mv_1@key=201 +POSTHOOK: Output: default@partition_mv_1@key=202 +POSTHOOK: Output: default@partition_mv_1@key=203 +POSTHOOK: Output: default@partition_mv_1@key=205 +POSTHOOK: Output: default@partition_mv_1@key=207 +POSTHOOK: Output: default@partition_mv_1@key=208 +POSTHOOK: Output: default@partition_mv_1@key=209 +POSTHOOK: Output: default@partition_mv_1@key=213 +POSTHOOK: Output: default@partition_mv_1@key=214 +POSTHOOK: Output: default@partition_mv_1@key=216 +POSTHOOK: Output: default@partition_mv_1@key=217 +POSTHOOK: Output: default@partition_mv_1@key=218 +POSTHOOK: Output: default@partition_mv_1@key=219 +POSTHOOK: Output: default@partition_mv_1@key=221 +POSTHOOK: Output: default@partition_mv_1@key=222 +POSTHOOK: Output: default@partition_mv_1@key=223 +POSTHOOK: Output: default@partition_mv_1@key=224 +POSTHOOK: Output: default@partition_mv_1@key=226 +POSTHOOK: Output: default@partition_mv_1@key=228 +POSTHOOK: Output: default@partition_mv_1@key=229 +POSTHOOK: Output: default@partition_mv_1@key=230 +POSTHOOK: Output: default@partition_mv_1@key=233 +POSTHOOK: Output: default@partition_mv_1@key=235 +POSTHOOK: Output: default@partition_mv_1@key=237 +POSTHOOK: Output: default@partition_mv_1@key=238 +POSTHOOK: Output: default@partition_mv_1@key=239 +POSTHOOK: Output: default@partition_mv_1@key=241 +POSTHOOK: Output: default@partition_mv_1@key=242 +POSTHOOK: Output: default@partition_mv_1@key=244 +POSTHOOK: Output: default@partition_mv_1@key=247 +POSTHOOK: Output: default@partition_mv_1@key=248 +POSTHOOK: Output: default@partition_mv_1@key=249 +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=201).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=202).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=203).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=205).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=207).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=208).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=209).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=213).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=214).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=216).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=217).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=218).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=219).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=221).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=222).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=223).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=224).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=226).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=228).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=229).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=230).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=233).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=235).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=237).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=238).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=239).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=241).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=242).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=244).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=247).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=248).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_1 PARTITION(key=249).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM partition_mv_1 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_mv_1 +PREHOOK: Input: default@partition_mv_1@key=238 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM partition_mv_1 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_mv_1 +POSTHOOK: Input: default@partition_mv_1@key=238 +#### A masked pattern was here #### +val_238 238 +val_238 238 +val_238_n 238 +PREHOOK: query: SELECT * FROM partition_mv_2 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_mv_2 +PREHOOK: Input: default@partition_mv_2@value=val_201 +PREHOOK: Input: default@partition_mv_2@value=val_202 +PREHOOK: Input: default@partition_mv_2@value=val_203 +PREHOOK: Input: default@partition_mv_2@value=val_205 +PREHOOK: Input: default@partition_mv_2@value=val_207 +PREHOOK: Input: default@partition_mv_2@value=val_208 +PREHOOK: Input: default@partition_mv_2@value=val_209 +PREHOOK: Input: default@partition_mv_2@value=val_213 +PREHOOK: Input: default@partition_mv_2@value=val_214 +PREHOOK: Input: default@partition_mv_2@value=val_216 +PREHOOK: Input: default@partition_mv_2@value=val_217 +PREHOOK: Input: default@partition_mv_2@value=val_218 +PREHOOK: Input: default@partition_mv_2@value=val_219 +PREHOOK: Input: default@partition_mv_2@value=val_221 +PREHOOK: Input: default@partition_mv_2@value=val_222 +PREHOOK: Input: default@partition_mv_2@value=val_223 +PREHOOK: Input: default@partition_mv_2@value=val_224 +PREHOOK: Input: default@partition_mv_2@value=val_226 +PREHOOK: Input: default@partition_mv_2@value=val_228 +PREHOOK: Input: default@partition_mv_2@value=val_229 +PREHOOK: Input: default@partition_mv_2@value=val_230 +PREHOOK: Input: default@partition_mv_2@value=val_233 +PREHOOK: Input: default@partition_mv_2@value=val_235 +PREHOOK: Input: default@partition_mv_2@value=val_237 +PREHOOK: Input: default@partition_mv_2@value=val_238 +PREHOOK: Input: default@partition_mv_2@value=val_239 +PREHOOK: Input: default@partition_mv_2@value=val_241 +PREHOOK: Input: default@partition_mv_2@value=val_242 +PREHOOK: Input: default@partition_mv_2@value=val_244 +PREHOOK: Input: default@partition_mv_2@value=val_247 +PREHOOK: Input: default@partition_mv_2@value=val_248 +PREHOOK: Input: default@partition_mv_2@value=val_249 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM partition_mv_2 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_mv_2 +POSTHOOK: Input: default@partition_mv_2@value=val_201 +POSTHOOK: Input: default@partition_mv_2@value=val_202 +POSTHOOK: Input: default@partition_mv_2@value=val_203 +POSTHOOK: Input: default@partition_mv_2@value=val_205 +POSTHOOK: Input: default@partition_mv_2@value=val_207 +POSTHOOK: Input: default@partition_mv_2@value=val_208 +POSTHOOK: Input: default@partition_mv_2@value=val_209 +POSTHOOK: Input: default@partition_mv_2@value=val_213 +POSTHOOK: Input: default@partition_mv_2@value=val_214 +POSTHOOK: Input: default@partition_mv_2@value=val_216 +POSTHOOK: Input: default@partition_mv_2@value=val_217 +POSTHOOK: Input: default@partition_mv_2@value=val_218 +POSTHOOK: Input: default@partition_mv_2@value=val_219 +POSTHOOK: Input: default@partition_mv_2@value=val_221 +POSTHOOK: Input: default@partition_mv_2@value=val_222 +POSTHOOK: Input: default@partition_mv_2@value=val_223 +POSTHOOK: Input: default@partition_mv_2@value=val_224 +POSTHOOK: Input: default@partition_mv_2@value=val_226 +POSTHOOK: Input: default@partition_mv_2@value=val_228 +POSTHOOK: Input: default@partition_mv_2@value=val_229 +POSTHOOK: Input: default@partition_mv_2@value=val_230 +POSTHOOK: Input: default@partition_mv_2@value=val_233 +POSTHOOK: Input: default@partition_mv_2@value=val_235 +POSTHOOK: Input: default@partition_mv_2@value=val_237 +POSTHOOK: Input: default@partition_mv_2@value=val_238 +POSTHOOK: Input: default@partition_mv_2@value=val_239 +POSTHOOK: Input: default@partition_mv_2@value=val_241 +POSTHOOK: Input: default@partition_mv_2@value=val_242 +POSTHOOK: Input: default@partition_mv_2@value=val_244 +POSTHOOK: Input: default@partition_mv_2@value=val_247 +POSTHOOK: Input: default@partition_mv_2@value=val_248 +POSTHOOK: Input: default@partition_mv_2@value=val_249 +#### A masked pattern was here #### +238 val_238 +238 val_238 +PREHOOK: query: CREATE TABLE src_txn_2 stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src_txn_2 +POSTHOOK: query: CREATE TABLE src_txn_2 stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_txn_2 +POSTHOOK: Lineage: src_txn_2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_txn_2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: CREATE MATERIALIZED VIEW partition_mv_3 PARTITIONED ON (key) AS +SELECT src_txn.value, src_txn.key FROM src_txn, src_txn_2 +WHERE src_txn.key = src_txn_2.key + AND src_txn.key > 200 AND src_txn.key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: database:default +PREHOOK: Output: default@partition_mv_3 +PREHOOK: Output: default@partition_mv_3 +POSTHOOK: query: CREATE MATERIALIZED VIEW partition_mv_3 PARTITIONED ON (key) AS +SELECT src_txn.value, src_txn.key FROM src_txn, src_txn_2 +WHERE src_txn.key = src_txn_2.key + AND src_txn.key > 200 AND src_txn.key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@partition_mv_3 +POSTHOOK: Output: default@partition_mv_3@key=201 +POSTHOOK: Output: default@partition_mv_3@key=202 +POSTHOOK: Output: default@partition_mv_3@key=203 +POSTHOOK: Output: default@partition_mv_3@key=205 +POSTHOOK: Output: default@partition_mv_3@key=207 +POSTHOOK: Output: default@partition_mv_3@key=208 +POSTHOOK: Output: default@partition_mv_3@key=209 +POSTHOOK: Output: default@partition_mv_3@key=213 +POSTHOOK: Output: default@partition_mv_3@key=214 +POSTHOOK: Output: default@partition_mv_3@key=216 +POSTHOOK: Output: default@partition_mv_3@key=217 +POSTHOOK: Output: default@partition_mv_3@key=218 +POSTHOOK: Output: default@partition_mv_3@key=219 +POSTHOOK: Output: default@partition_mv_3@key=221 +POSTHOOK: Output: default@partition_mv_3@key=222 +POSTHOOK: Output: default@partition_mv_3@key=223 +POSTHOOK: Output: default@partition_mv_3@key=224 +POSTHOOK: Output: default@partition_mv_3@key=226 +POSTHOOK: Output: default@partition_mv_3@key=228 +POSTHOOK: Output: default@partition_mv_3@key=229 +POSTHOOK: Output: default@partition_mv_3@key=230 +POSTHOOK: Output: default@partition_mv_3@key=233 +POSTHOOK: Output: default@partition_mv_3@key=235 +POSTHOOK: Output: default@partition_mv_3@key=237 +POSTHOOK: Output: default@partition_mv_3@key=238 +POSTHOOK: Output: default@partition_mv_3@key=239 +POSTHOOK: Output: default@partition_mv_3@key=241 +POSTHOOK: Output: default@partition_mv_3@key=242 +POSTHOOK: Output: default@partition_mv_3@key=244 +POSTHOOK: Output: default@partition_mv_3@key=247 +POSTHOOK: Output: default@partition_mv_3@key=248 +POSTHOOK: Output: default@partition_mv_3@key=249 +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=201).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=202).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=203).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=205).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=207).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=208).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=209).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=213).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=214).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=216).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=217).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=218).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=219).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=221).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=222).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=223).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=224).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=226).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=228).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=229).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=230).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=233).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=235).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=237).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=238).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=239).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=241).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=242).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=244).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=247).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=248).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=249).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@src_txn +POSTHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SCRIPT [] +POSTHOOK: Lineage: src_txn.value SCRIPT [] +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW partition_mv_3 REBUILD +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW partition_mv_3 REBUILD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 502 Data size: 175904 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) < 250.0D) and (UDFToDouble(key) > 200.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 19272 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 19272 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 19272 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: src_txn_2 + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) < 250.0D) and (UDFToDouble(key) > 200.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 9634 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 55 Data size: 9634 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 9634 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 60 Data size: 21199 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 60 Data size: 21199 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 60 Data size: 21199 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.partition_mv_3 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: value, key + Statistics: Num rows: 60 Data size: 21199 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(value, 'hll') + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 60 Data size: 21199 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 60 Data size: 21199 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 30 Data size: 10599 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: struct), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 30 Data size: 10599 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 30 Data size: 10599 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + key + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.partition_mv_3 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value + Column Types: string + Table: default.partition_mv_3 + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW partition_mv_3 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: default@partition_mv_3 +POSTHOOK: query: ALTER MATERIALIZED VIEW partition_mv_3 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: default@partition_mv_3@key=201 +POSTHOOK: Output: default@partition_mv_3@key=202 +POSTHOOK: Output: default@partition_mv_3@key=203 +POSTHOOK: Output: default@partition_mv_3@key=205 +POSTHOOK: Output: default@partition_mv_3@key=207 +POSTHOOK: Output: default@partition_mv_3@key=208 +POSTHOOK: Output: default@partition_mv_3@key=209 +POSTHOOK: Output: default@partition_mv_3@key=213 +POSTHOOK: Output: default@partition_mv_3@key=214 +POSTHOOK: Output: default@partition_mv_3@key=216 +POSTHOOK: Output: default@partition_mv_3@key=217 +POSTHOOK: Output: default@partition_mv_3@key=218 +POSTHOOK: Output: default@partition_mv_3@key=219 +POSTHOOK: Output: default@partition_mv_3@key=221 +POSTHOOK: Output: default@partition_mv_3@key=222 +POSTHOOK: Output: default@partition_mv_3@key=223 +POSTHOOK: Output: default@partition_mv_3@key=224 +POSTHOOK: Output: default@partition_mv_3@key=226 +POSTHOOK: Output: default@partition_mv_3@key=228 +POSTHOOK: Output: default@partition_mv_3@key=229 +POSTHOOK: Output: default@partition_mv_3@key=230 +POSTHOOK: Output: default@partition_mv_3@key=233 +POSTHOOK: Output: default@partition_mv_3@key=235 +POSTHOOK: Output: default@partition_mv_3@key=237 +POSTHOOK: Output: default@partition_mv_3@key=238 +POSTHOOK: Output: default@partition_mv_3@key=239 +POSTHOOK: Output: default@partition_mv_3@key=241 +POSTHOOK: Output: default@partition_mv_3@key=242 +POSTHOOK: Output: default@partition_mv_3@key=244 +POSTHOOK: Output: default@partition_mv_3@key=247 +POSTHOOK: Output: default@partition_mv_3@key=248 +POSTHOOK: Output: default@partition_mv_3@key=249 +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=201).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=202).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=203).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=205).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=207).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=208).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=209).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=213).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=214).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=216).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=217).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=218).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=219).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=221).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=222).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=223).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=224).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=226).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=228).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=229).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=230).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=233).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=235).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=237).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=238).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=239).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=241).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=242).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=244).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=247).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=248).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: partition_mv_3 PARTITION(key=249).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM partition_mv_3 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_mv_3 +PREHOOK: Input: default@partition_mv_3@key=238 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM partition_mv_3 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_mv_3 +POSTHOOK: Input: default@partition_mv_3@key=238 +#### A masked pattern was here #### +val_238 238 +val_238 238 +val_238 238 +val_238 238 +val_238_n2 238 +val_238_n2 238 +val_238_n 238 +val_238_n 238 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_part_1.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_part_1.q.out new file mode 100644 index 0000000000..492bb226fd --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_part_1.q.out @@ -0,0 +1,1121 @@ +PREHOOK: query: create table emps_n30 ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@emps_n30 +POSTHOOK: query: create table emps_n30 ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@emps_n30 +PREHOOK: query: insert into emps_n30 values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@emps_n30 +POSTHOOK: query: insert into emps_n30 values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@emps_n30 +POSTHOOK: Lineage: emps_n30.commission SCRIPT [] +POSTHOOK: Lineage: emps_n30.deptno SCRIPT [] +POSTHOOK: Lineage: emps_n30.empid SCRIPT [] +POSTHOOK: Lineage: emps_n30.name SCRIPT [] +POSTHOOK: Lineage: emps_n30.salary SCRIPT [] +PREHOOK: query: analyze table emps_n30 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@emps_n30 +PREHOOK: Output: default@emps_n30 +#### A masked pattern was here #### +POSTHOOK: query: analyze table emps_n30 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@emps_n30 +POSTHOOK: Output: default@emps_n30 +#### A masked pattern was here #### +PREHOOK: query: create table depts_n20 ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@depts_n20 +POSTHOOK: query: create table depts_n20 ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@depts_n20 +PREHOOK: query: insert into depts_n20 values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@depts_n20 +POSTHOOK: query: insert into depts_n20 values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@depts_n20 +POSTHOOK: Lineage: depts_n20.deptno SCRIPT [] +POSTHOOK: Lineage: depts_n20.locationid SCRIPT [] +POSTHOOK: Lineage: depts_n20.name SCRIPT [] +PREHOOK: query: analyze table depts_n20 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@depts_n20 +PREHOOK: Output: default@depts_n20 +#### A masked pattern was here #### +POSTHOOK: query: analyze table depts_n20 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@depts_n20 +POSTHOOK: Output: default@depts_n20 +#### A masked pattern was here #### +PREHOOK: query: create table dependents_n20 ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dependents_n20 +POSTHOOK: query: create table dependents_n20 ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dependents_n20 +PREHOOK: query: insert into dependents_n20 values (10, 'Michael'), (10, 'Jane') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dependents_n20 +POSTHOOK: query: insert into dependents_n20 values (10, 'Michael'), (10, 'Jane') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dependents_n20 +POSTHOOK: Lineage: dependents_n20.empid SCRIPT [] +POSTHOOK: Lineage: dependents_n20.name SCRIPT [] +PREHOOK: query: analyze table dependents_n20 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@dependents_n20 +PREHOOK: Output: default@dependents_n20 +#### A masked pattern was here #### +POSTHOOK: query: analyze table dependents_n20 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@dependents_n20 +POSTHOOK: Output: default@dependents_n20 +#### A masked pattern was here #### +PREHOOK: query: create table locations_n20 ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@locations_n20 +POSTHOOK: query: create table locations_n20 ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@locations_n20 +PREHOOK: query: insert into locations_n20 values (10, 'San Francisco'), (10, 'San Diego') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@locations_n20 +POSTHOOK: query: insert into locations_n20 values (10, 'San Francisco'), (10, 'San Diego') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@locations_n20 +POSTHOOK: Lineage: locations_n20.locationid SCRIPT [] +POSTHOOK: Lineage: locations_n20.name SCRIPT [] +PREHOOK: query: analyze table locations_n20 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@locations_n20 +PREHOOK: Output: default@locations_n20 +#### A masked pattern was here #### +POSTHOOK: query: analyze table locations_n20 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@locations_n20 +POSTHOOK: Output: default@locations_n20 +#### A masked pattern was here #### +PREHOOK: query: alter table emps_n30 add constraint pk1 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps_n30 add constraint pk1 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts_n20 add constraint pk2 primary key (deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts_n20 add constraint pk2 primary key (deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table dependents_n20 add constraint pk3 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table dependents_n20 add constraint pk3 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table locations_n20 add constraint pk4 primary key (locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table locations_n20 add constraint pk4 primary key (locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table emps_n30 add constraint fk1 foreign key (deptno) references depts_n20(deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps_n30 add constraint fk1 foreign key (deptno) references depts_n20(deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts_n20 add constraint fk2 foreign key (locationid) references locations_n20(locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts_n20 add constraint fk2 foreign key (locationid) references locations_n20(locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: create materialized view mv1_part_n2 partitioned on (deptno) as +select * from emps_n30 where empid < 150 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps_n30 +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2 +POSTHOOK: query: create materialized view mv1_part_n2 partitioned on (deptno) as +select * from emps_n30 where empid < 150 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps_n30 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2@deptno=10 +POSTHOOK: Lineage: mv1_part_n2 PARTITION(deptno=10).commission SIMPLE [(emps_n30)emps_n30.FieldSchema(name:commission, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(deptno=10).empid SIMPLE [(emps_n30)emps_n30.FieldSchema(name:empid, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(deptno=10).name SIMPLE [(emps_n30)emps_n30.FieldSchema(name:name, type:varchar(256), comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(deptno=10).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +PREHOOK: query: analyze table mv1_part_n2 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Input: default@mv1_part_n2@deptno=10 +PREHOOK: Output: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2@deptno=10 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1_part_n2 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Input: default@mv1_part_n2@deptno=10 +POSTHOOK: Output: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2@deptno=10 +#### A masked pattern was here #### +PREHOOK: query: explain +select * +from (select * from emps_n30 where empid < 120) t +join depts_n20 using (deptno) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * +from (select * from emps_n30 where empid < 120) t +join depts_n20 using (deptno) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1_part_n2 + filterExpr: ((empid < 120) and name is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 315 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((empid < 120) and name is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 315 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), UDFToInteger(name) (type: int), CAST( salary AS varchar(256)) (type: varchar(256)), UDFToFloat(commission) (type: float), deptno (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 3 Data size: 1068 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 3 Data size: 1068 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: varchar(256)), _col3 (type: float), _col4 (type: int) + Execution mode: llap + LLAP IO: all inputs + Map 3 + Map Operator Tree: + TableScan + alias: depts_n20 + Statistics: Num rows: 3 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int), name (type: varchar(256)), locationid (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 3 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 291 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: varchar(256)), _col2 (type: int) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 3 Data size: 1347 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col5 (type: int), _col0 (type: int), _col2 (type: varchar(256)), _col3 (type: float), _col4 (type: int), _col6 (type: varchar(256)), _col7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 3 Data size: 1347 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 1347 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * +from (select * from emps_n30 where empid < 120) t +join depts_n20 using (deptno) +PREHOOK: type: QUERY +PREHOOK: Input: default@depts_n20 +PREHOOK: Input: default@emps_n30 +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Input: default@mv1_part_n2@deptno=10 +#### A masked pattern was here #### +POSTHOOK: query: select * +from (select * from emps_n30 where empid < 120) t +join depts_n20 using (deptno) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts_n20 +POSTHOOK: Input: default@emps_n30 +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Input: default@mv1_part_n2@deptno=10 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1_part_n2 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2 +POSTHOOK: query: drop materialized view mv1_part_n2 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2 +PREHOOK: query: create materialized view mv1_part_n2 partitioned on (deptno) as +select deptno, name, salary, commission +from emps_n30 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps_n30 +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2 +POSTHOOK: query: create materialized view mv1_part_n2 partitioned on (deptno) as +select deptno, name, salary, commission +from emps_n30 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps_n30 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2@deptno=10 +POSTHOOK: Output: default@mv1_part_n2@deptno=20 +POSTHOOK: Lineage: mv1_part_n2 PARTITION(deptno=10).commission SIMPLE [(emps_n30)emps_n30.FieldSchema(name:commission, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(deptno=10).name SIMPLE [(emps_n30)emps_n30.FieldSchema(name:name, type:varchar(256), comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(deptno=10).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(deptno=20).commission SIMPLE [(emps_n30)emps_n30.FieldSchema(name:commission, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(deptno=20).name SIMPLE [(emps_n30)emps_n30.FieldSchema(name:name, type:varchar(256), comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(deptno=20).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +PREHOOK: query: analyze table mv1_part_n2 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Input: default@mv1_part_n2@deptno=10 +PREHOOK: Input: default@mv1_part_n2@deptno=20 +PREHOOK: Output: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2@deptno=10 +PREHOOK: Output: default@mv1_part_n2@deptno=20 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1_part_n2 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Input: default@mv1_part_n2@deptno=10 +POSTHOOK: Input: default@mv1_part_n2@deptno=20 +POSTHOOK: Output: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2@deptno=10 +POSTHOOK: Output: default@mv1_part_n2@deptno=20 +#### A masked pattern was here #### +PREHOOK: query: explain +select emps_n30.name, emps_n30.salary, emps_n30.commission +from emps_n30 +join depts_n20 using (deptno) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select emps_n30.name, emps_n30.salary, emps_n30.commission +from emps_n30 +join depts_n20 using (deptno) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1_part_n2 + filterExpr: name is not null (type: boolean) + Statistics: Num rows: 5 Data size: 505 Basic stats: COMPLETE Column stats: PARTIAL + Filter Operator + predicate: name is not null (type: boolean) + Statistics: Num rows: 5 Data size: 505 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: UDFToInteger(name) (type: int), CAST( salary AS varchar(256)) (type: varchar(256)), UDFToFloat(commission) (type: float), deptno (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 5 Data size: 1760 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 1760 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: varchar(256)), _col2 (type: float), _col3 (type: int) + Execution mode: llap + LLAP IO: all inputs + Map 3 + Map Operator Tree: + TableScan + alias: depts_n20 + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1, _col2, _col3 + Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col1 (type: varchar(256)), _col2 (type: float), _col3 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select emps_n30.name, emps_n30.salary, emps_n30.commission +from emps_n30 +join depts_n20 using (deptno) +PREHOOK: type: QUERY +PREHOOK: Input: default@depts_n20 +PREHOOK: Input: default@emps_n30 +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Input: default@mv1_part_n2@deptno=10 +PREHOOK: Input: default@mv1_part_n2@deptno=20 +#### A masked pattern was here #### +POSTHOOK: query: select emps_n30.name, emps_n30.salary, emps_n30.commission +from emps_n30 +join depts_n20 using (deptno) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts_n20 +POSTHOOK: Input: default@emps_n30 +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Input: default@mv1_part_n2@deptno=10 +POSTHOOK: Input: default@mv1_part_n2@deptno=20 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1_part_n2 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2 +POSTHOOK: query: drop materialized view mv1_part_n2 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2 +PREHOOK: query: create materialized view mv1_part_n2 partitioned on (deptno) as +select * from emps_n30 where empid < 200 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps_n30 +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2 +POSTHOOK: query: create materialized view mv1_part_n2 partitioned on (deptno) as +select * from emps_n30 where empid < 200 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps_n30 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2@deptno=10 +POSTHOOK: Lineage: mv1_part_n2 PARTITION(deptno=10).commission SIMPLE [(emps_n30)emps_n30.FieldSchema(name:commission, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(deptno=10).empid SIMPLE [(emps_n30)emps_n30.FieldSchema(name:empid, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(deptno=10).name SIMPLE [(emps_n30)emps_n30.FieldSchema(name:name, type:varchar(256), comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(deptno=10).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +PREHOOK: query: analyze table mv1_part_n2 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Input: default@mv1_part_n2@deptno=10 +PREHOOK: Output: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2@deptno=10 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1_part_n2 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Input: default@mv1_part_n2@deptno=10 +POSTHOOK: Output: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2@deptno=10 +#### A masked pattern was here #### +PREHOOK: query: explain +select * from emps_n30 where empid > 120 +union all select * from emps_n30 where empid < 150 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from emps_n30 where empid > 120 +union all select * from emps_n30 where empid < 150 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: emps_n30 + filterExpr: (empid > 120) (type: boolean) + Statistics: Num rows: 5 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (empid > 120) (type: boolean) + Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), deptno (type: int), name (type: varchar(256)), salary (type: float), commission (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 1530 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: may be used (ACID table) + Map 3 + Map Operator Tree: + TableScan + alias: default.mv1_part_n2 + filterExpr: (empid < 150) (type: boolean) + Statistics: Num rows: 4 Data size: 420 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (empid < 150) (type: boolean) + Statistics: Num rows: 4 Data size: 420 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int), UDFToInteger(name) (type: int), CAST( salary AS varchar(256)) (type: varchar(256)), UDFToFloat(commission) (type: float), deptno (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 1530 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: all inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from emps_n30 where empid > 120 +union all select * from emps_n30 where empid < 150 +PREHOOK: type: QUERY +PREHOOK: Input: default@emps_n30 +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Input: default@mv1_part_n2@deptno=10 +#### A masked pattern was here #### +POSTHOOK: query: select * from emps_n30 where empid > 120 +union all select * from emps_n30 where empid < 150 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps_n30 +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Input: default@mv1_part_n2@deptno=10 +#### A masked pattern was here #### +100 NULL 10000.0 1000.0 10 +110 NULL 10000.0 250.0 10 +110 NULL 10000.0 250.0 10 +150 10 Sebastian 7000.0 NULL +200 20 Eric 8000.0 500 +PREHOOK: query: drop materialized view mv1_part_n2 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2 +POSTHOOK: query: drop materialized view mv1_part_n2 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2 +PREHOOK: query: create materialized view mv1_part_n2 partitioned on (name) as +select name, salary from emps_n30 group by name, salary +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps_n30 +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2 +POSTHOOK: query: create materialized view mv1_part_n2 partitioned on (name) as +select name, salary from emps_n30 group by name, salary +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps_n30 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2@name=Bill +POSTHOOK: Output: default@mv1_part_n2@name=Eric +POSTHOOK: Output: default@mv1_part_n2@name=Sebastian +POSTHOOK: Output: default@mv1_part_n2@name=Theodore +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Bill).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Eric).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Sebastian).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Theodore).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +PREHOOK: query: analyze table mv1_part_n2 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Input: default@mv1_part_n2@name=Bill +PREHOOK: Input: default@mv1_part_n2@name=Eric +PREHOOK: Input: default@mv1_part_n2@name=Sebastian +PREHOOK: Input: default@mv1_part_n2@name=Theodore +PREHOOK: Output: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2@name=Bill +PREHOOK: Output: default@mv1_part_n2@name=Eric +PREHOOK: Output: default@mv1_part_n2@name=Sebastian +PREHOOK: Output: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1_part_n2 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Input: default@mv1_part_n2@name=Bill +POSTHOOK: Input: default@mv1_part_n2@name=Eric +POSTHOOK: Input: default@mv1_part_n2@name=Sebastian +POSTHOOK: Input: default@mv1_part_n2@name=Theodore +POSTHOOK: Output: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2@name=Bill +POSTHOOK: Output: default@mv1_part_n2@name=Eric +POSTHOOK: Output: default@mv1_part_n2@name=Sebastian +POSTHOOK: Output: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +PREHOOK: query: explain +select name, salary from emps_n30 group by name, salary +PREHOOK: type: QUERY +POSTHOOK: query: explain +select name, salary from emps_n30 group by name, salary +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1_part_n2 + Select Operator + expressions: CAST( salary AS varchar(256)) (type: varchar(256)), UDFToFloat(name) (type: float) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: select name, salary from emps_n30 group by name, salary +PREHOOK: type: QUERY +PREHOOK: Input: default@emps_n30 +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Input: default@mv1_part_n2@name=Bill +PREHOOK: Input: default@mv1_part_n2@name=Eric +PREHOOK: Input: default@mv1_part_n2@name=Sebastian +PREHOOK: Input: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +POSTHOOK: query: select name, salary from emps_n30 group by name, salary +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps_n30 +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Input: default@mv1_part_n2@name=Bill +POSTHOOK: Input: default@mv1_part_n2@name=Eric +POSTHOOK: Input: default@mv1_part_n2@name=Sebastian +POSTHOOK: Input: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +10000.0 NULL +10000.0 NULL +7000.0 NULL +8000.0 NULL +PREHOOK: query: drop materialized view mv1_part_n2 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2 +POSTHOOK: query: drop materialized view mv1_part_n2 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2 +PREHOOK: query: create materialized view mv1_part_n2 partitioned on (name) as +select name, salary from emps_n30 group by name, salary +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps_n30 +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2 +POSTHOOK: query: create materialized view mv1_part_n2 partitioned on (name) as +select name, salary from emps_n30 group by name, salary +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps_n30 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2@name=Bill +POSTHOOK: Output: default@mv1_part_n2@name=Eric +POSTHOOK: Output: default@mv1_part_n2@name=Sebastian +POSTHOOK: Output: default@mv1_part_n2@name=Theodore +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Bill).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Eric).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Sebastian).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Theodore).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +PREHOOK: query: analyze table mv1_part_n2 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Input: default@mv1_part_n2@name=Bill +PREHOOK: Input: default@mv1_part_n2@name=Eric +PREHOOK: Input: default@mv1_part_n2@name=Sebastian +PREHOOK: Input: default@mv1_part_n2@name=Theodore +PREHOOK: Output: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2@name=Bill +PREHOOK: Output: default@mv1_part_n2@name=Eric +PREHOOK: Output: default@mv1_part_n2@name=Sebastian +PREHOOK: Output: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1_part_n2 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Input: default@mv1_part_n2@name=Bill +POSTHOOK: Input: default@mv1_part_n2@name=Eric +POSTHOOK: Input: default@mv1_part_n2@name=Sebastian +POSTHOOK: Input: default@mv1_part_n2@name=Theodore +POSTHOOK: Output: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2@name=Bill +POSTHOOK: Output: default@mv1_part_n2@name=Eric +POSTHOOK: Output: default@mv1_part_n2@name=Sebastian +POSTHOOK: Output: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +PREHOOK: query: explain +select name from emps_n30 group by name +PREHOOK: type: QUERY +POSTHOOK: query: explain +select name from emps_n30 group by name +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1_part_n2 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: CAST( salary AS varchar(256)) (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: varchar(256)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 680 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 680 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 680 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 680 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select name from emps_n30 group by name +PREHOOK: type: QUERY +PREHOOK: Input: default@emps_n30 +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Input: default@mv1_part_n2@name=Bill +PREHOOK: Input: default@mv1_part_n2@name=Eric +PREHOOK: Input: default@mv1_part_n2@name=Sebastian +PREHOOK: Input: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +POSTHOOK: query: select name from emps_n30 group by name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps_n30 +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Input: default@mv1_part_n2@name=Bill +POSTHOOK: Input: default@mv1_part_n2@name=Eric +POSTHOOK: Input: default@mv1_part_n2@name=Sebastian +POSTHOOK: Input: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +10000.0 +7000.0 +8000.0 +PREHOOK: query: drop materialized view mv1_part_n2 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2 +POSTHOOK: query: drop materialized view mv1_part_n2 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2 +PREHOOK: query: create materialized view mv1_part_n2 partitioned on (name) as +select name, salary from emps_n30 where deptno = 10 group by name, salary +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps_n30 +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2 +POSTHOOK: query: create materialized view mv1_part_n2 partitioned on (name) as +select name, salary from emps_n30 where deptno = 10 group by name, salary +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps_n30 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2@name=Bill +POSTHOOK: Output: default@mv1_part_n2@name=Sebastian +POSTHOOK: Output: default@mv1_part_n2@name=Theodore +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Bill).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Sebastian).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Theodore).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +PREHOOK: query: analyze table mv1_part_n2 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Input: default@mv1_part_n2@name=Bill +PREHOOK: Input: default@mv1_part_n2@name=Sebastian +PREHOOK: Input: default@mv1_part_n2@name=Theodore +PREHOOK: Output: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2@name=Bill +PREHOOK: Output: default@mv1_part_n2@name=Sebastian +PREHOOK: Output: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1_part_n2 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Input: default@mv1_part_n2@name=Bill +POSTHOOK: Input: default@mv1_part_n2@name=Sebastian +POSTHOOK: Input: default@mv1_part_n2@name=Theodore +POSTHOOK: Output: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2@name=Bill +POSTHOOK: Output: default@mv1_part_n2@name=Sebastian +POSTHOOK: Output: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +PREHOOK: query: explain +select name from emps_n30 where deptno = 10 group by name +PREHOOK: type: QUERY +POSTHOOK: query: explain +select name from emps_n30 where deptno = 10 group by name +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1_part_n2 + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: CAST( salary AS varchar(256)) (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: varchar(256)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: PARTIAL + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select name from emps_n30 where deptno = 10 group by name +PREHOOK: type: QUERY +PREHOOK: Input: default@emps_n30 +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Input: default@mv1_part_n2@name=Bill +PREHOOK: Input: default@mv1_part_n2@name=Sebastian +PREHOOK: Input: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +POSTHOOK: query: select name from emps_n30 where deptno = 10 group by name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps_n30 +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Input: default@mv1_part_n2@name=Bill +POSTHOOK: Input: default@mv1_part_n2@name=Sebastian +POSTHOOK: Input: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +10000.0 +7000.0 +PREHOOK: query: drop materialized view mv1_part_n2 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2 +POSTHOOK: query: drop materialized view mv1_part_n2 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2 +PREHOOK: query: create materialized view mv1_part_n2 partitioned on (name) as +select name, salary, count(*) as c, sum(empid) as s +from emps_n30 group by name, salary +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@emps_n30 +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2 +POSTHOOK: query: create materialized view mv1_part_n2 partitioned on (name) as +select name, salary, count(*) as c, sum(empid) as s +from emps_n30 group by name, salary +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@emps_n30 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2@name=Bill +POSTHOOK: Output: default@mv1_part_n2@name=Eric +POSTHOOK: Output: default@mv1_part_n2@name=Sebastian +POSTHOOK: Output: default@mv1_part_n2@name=Theodore +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Bill).c EXPRESSION [(emps_n30)emps_n30.null, ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Bill).s EXPRESSION [(emps_n30)emps_n30.FieldSchema(name:empid, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Bill).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Eric).c EXPRESSION [(emps_n30)emps_n30.null, ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Eric).s EXPRESSION [(emps_n30)emps_n30.FieldSchema(name:empid, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Eric).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Sebastian).c EXPRESSION [(emps_n30)emps_n30.null, ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Sebastian).s EXPRESSION [(emps_n30)emps_n30.FieldSchema(name:empid, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Sebastian).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Theodore).c EXPRESSION [(emps_n30)emps_n30.null, ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Theodore).s EXPRESSION [(emps_n30)emps_n30.FieldSchema(name:empid, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n2 PARTITION(name=Theodore).salary SIMPLE [(emps_n30)emps_n30.FieldSchema(name:salary, type:float, comment:null), ] +PREHOOK: query: analyze table mv1_part_n2 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Input: default@mv1_part_n2@name=Bill +PREHOOK: Input: default@mv1_part_n2@name=Eric +PREHOOK: Input: default@mv1_part_n2@name=Sebastian +PREHOOK: Input: default@mv1_part_n2@name=Theodore +PREHOOK: Output: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2@name=Bill +PREHOOK: Output: default@mv1_part_n2@name=Eric +PREHOOK: Output: default@mv1_part_n2@name=Sebastian +PREHOOK: Output: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1_part_n2 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Input: default@mv1_part_n2@name=Bill +POSTHOOK: Input: default@mv1_part_n2@name=Eric +POSTHOOK: Input: default@mv1_part_n2@name=Sebastian +POSTHOOK: Input: default@mv1_part_n2@name=Theodore +POSTHOOK: Output: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2@name=Bill +POSTHOOK: Output: default@mv1_part_n2@name=Eric +POSTHOOK: Output: default@mv1_part_n2@name=Sebastian +POSTHOOK: Output: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +PREHOOK: query: explain +select name from emps_n30 group by name +PREHOOK: type: QUERY +POSTHOOK: query: explain +select name from emps_n30 group by name +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1_part_n2 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: CAST( salary AS varchar(256)) (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: varchar(256)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 680 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 680 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 680 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 680 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select name from emps_n30 group by name +PREHOOK: type: QUERY +PREHOOK: Input: default@emps_n30 +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Input: default@mv1_part_n2@name=Bill +PREHOOK: Input: default@mv1_part_n2@name=Eric +PREHOOK: Input: default@mv1_part_n2@name=Sebastian +PREHOOK: Input: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +POSTHOOK: query: select name from emps_n30 group by name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emps_n30 +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Input: default@mv1_part_n2@name=Bill +POSTHOOK: Input: default@mv1_part_n2@name=Eric +POSTHOOK: Input: default@mv1_part_n2@name=Sebastian +POSTHOOK: Input: default@mv1_part_n2@name=Theodore +#### A masked pattern was here #### +10000.0 +7000.0 +8000.0 +PREHOOK: query: drop materialized view mv1_part_n2 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1_part_n2 +PREHOOK: Output: default@mv1_part_n2 +POSTHOOK: query: drop materialized view mv1_part_n2 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1_part_n2 +POSTHOOK: Output: default@mv1_part_n2 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_part_2.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_part_2.q.out new file mode 100644 index 0000000000..e748ccb010 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_part_2.q.out @@ -0,0 +1,829 @@ +PREHOOK: query: create table emps_n00 ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@emps_n00 +POSTHOOK: query: create table emps_n00 ( + empid int, + deptno int, + name varchar(256), + salary float, + commission int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@emps_n00 +PREHOOK: query: insert into emps_n00 values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@emps_n00 +POSTHOOK: query: insert into emps_n00 values (100, 10, 'Bill', 10000, 1000), (200, 20, 'Eric', 8000, 500), + (150, 10, 'Sebastian', 7000, null), (110, 10, 'Theodore', 10000, 250), (110, 10, 'Bill', 10000, 250) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@emps_n00 +POSTHOOK: Lineage: emps_n00.commission SCRIPT [] +POSTHOOK: Lineage: emps_n00.deptno SCRIPT [] +POSTHOOK: Lineage: emps_n00.empid SCRIPT [] +POSTHOOK: Lineage: emps_n00.name SCRIPT [] +POSTHOOK: Lineage: emps_n00.salary SCRIPT [] +PREHOOK: query: analyze table emps_n00 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@emps_n00 +PREHOOK: Output: default@emps_n00 +#### A masked pattern was here #### +POSTHOOK: query: analyze table emps_n00 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@emps_n00 +POSTHOOK: Output: default@emps_n00 +#### A masked pattern was here #### +PREHOOK: query: create table depts_n00 ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@depts_n00 +POSTHOOK: query: create table depts_n00 ( + deptno int, + name varchar(256), + locationid int) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@depts_n00 +PREHOOK: query: insert into depts_n00 values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@depts_n00 +POSTHOOK: query: insert into depts_n00 values (10, 'Sales', 10), (30, 'Marketing', null), (20, 'HR', 20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@depts_n00 +POSTHOOK: Lineage: depts_n00.deptno SCRIPT [] +POSTHOOK: Lineage: depts_n00.locationid SCRIPT [] +POSTHOOK: Lineage: depts_n00.name SCRIPT [] +PREHOOK: query: analyze table depts_n00 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@depts_n00 +PREHOOK: Output: default@depts_n00 +#### A masked pattern was here #### +POSTHOOK: query: analyze table depts_n00 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@depts_n00 +POSTHOOK: Output: default@depts_n00 +#### A masked pattern was here #### +PREHOOK: query: create table dependents_n00 ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dependents_n00 +POSTHOOK: query: create table dependents_n00 ( + empid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dependents_n00 +PREHOOK: query: insert into dependents_n00 values (10, 'Michael'), (10, 'Jane') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@dependents_n00 +POSTHOOK: query: insert into dependents_n00 values (10, 'Michael'), (10, 'Jane') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@dependents_n00 +POSTHOOK: Lineage: dependents_n00.empid SCRIPT [] +POSTHOOK: Lineage: dependents_n00.name SCRIPT [] +PREHOOK: query: analyze table dependents_n00 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@dependents_n00 +PREHOOK: Output: default@dependents_n00 +#### A masked pattern was here #### +POSTHOOK: query: analyze table dependents_n00 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@dependents_n00 +POSTHOOK: Output: default@dependents_n00 +#### A masked pattern was here #### +PREHOOK: query: create table locations_n00 ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@locations_n00 +POSTHOOK: query: create table locations_n00 ( + locationid int, + name varchar(256)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@locations_n00 +PREHOOK: query: insert into locations_n00 values (10, 'San Francisco'), (10, 'San Diego') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@locations_n00 +POSTHOOK: query: insert into locations_n00 values (10, 'San Francisco'), (10, 'San Diego') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@locations_n00 +POSTHOOK: Lineage: locations_n00.locationid SCRIPT [] +POSTHOOK: Lineage: locations_n00.name SCRIPT [] +PREHOOK: query: analyze table locations_n00 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@locations_n00 +PREHOOK: Output: default@locations_n00 +#### A masked pattern was here #### +POSTHOOK: query: analyze table locations_n00 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@locations_n00 +POSTHOOK: Output: default@locations_n00 +#### A masked pattern was here #### +PREHOOK: query: alter table emps_n00 add constraint pk1 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps_n00 add constraint pk1 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts_n00 add constraint pk2 primary key (deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts_n00 add constraint pk2 primary key (deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table dependents_n00 add constraint pk3 primary key (empid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table dependents_n00 add constraint pk3 primary key (empid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table locations_n00 add constraint pk4 primary key (locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table locations_n00 add constraint pk4 primary key (locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table emps_n00 add constraint fk1 foreign key (deptno) references depts_n00(deptno) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table emps_n00 add constraint fk1 foreign key (deptno) references depts_n00(deptno) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: alter table depts_n00 add constraint fk2 foreign key (locationid) references locations_n00(locationid) disable novalidate rely +PREHOOK: type: ALTERTABLE_ADDCONSTRAINT +POSTHOOK: query: alter table depts_n00 add constraint fk2 foreign key (locationid) references locations_n00(locationid) disable novalidate rely +POSTHOOK: type: ALTERTABLE_ADDCONSTRAINT +PREHOOK: query: create materialized view mv1_part_n0 partitioned on (deptno) as +select empid, depts_n00.deptno as deptno from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 10 +group by empid, depts_n00.deptno +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts_n00 +PREHOOK: Input: default@emps_n00 +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0 +POSTHOOK: query: create materialized view mv1_part_n0 partitioned on (deptno) as +select empid, depts_n00.deptno as deptno from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 10 +group by empid, depts_n00.deptno +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts_n00 +POSTHOOK: Input: default@emps_n00 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0@deptno=20 +POSTHOOK: Lineage: mv1_part_n0 PARTITION(deptno=20).empid SIMPLE [(emps_n00)emps_n00.FieldSchema(name:empid, type:int, comment:null), ] +PREHOOK: query: analyze table mv1_part_n0 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Input: default@mv1_part_n0@deptno=20 +PREHOOK: Output: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1_part_n0 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Input: default@mv1_part_n0@deptno=20 +POSTHOOK: Output: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +PREHOOK: query: explain +select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1_part_n0 + filterExpr: (deptno > 20) (type: boolean) + Filter Operator + predicate: (deptno > 20) (type: boolean) + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@depts_n00 +PREHOOK: Input: default@emps_n00 +PREHOOK: Input: default@mv1_part_n0 +#### A masked pattern was here #### +POSTHOOK: query: select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts_n00 +POSTHOOK: Input: default@emps_n00 +POSTHOOK: Input: default@mv1_part_n0 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1_part_n0 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0 +POSTHOOK: query: drop materialized view mv1_part_n0 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0 +PREHOOK: query: create materialized view mv1_part_n0 partitioned on (deptno) as +select depts_n00.deptno as deptno, empid from depts_n00 +join emps_n00 using (deptno) where depts_n00.deptno > 10 +group by empid, depts_n00.deptno +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts_n00 +PREHOOK: Input: default@emps_n00 +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0 +POSTHOOK: query: create materialized view mv1_part_n0 partitioned on (deptno) as +select depts_n00.deptno as deptno, empid from depts_n00 +join emps_n00 using (deptno) where depts_n00.deptno > 10 +group by empid, depts_n00.deptno +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts_n00 +POSTHOOK: Input: default@emps_n00 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0@deptno=20 +POSTHOOK: Lineage: mv1_part_n0 PARTITION(deptno=20).empid SIMPLE [(emps_n00)emps_n00.FieldSchema(name:empid, type:int, comment:null), ] +PREHOOK: query: analyze table mv1_part_n0 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Input: default@mv1_part_n0@deptno=20 +PREHOOK: Output: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1_part_n0 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Input: default@mv1_part_n0@deptno=20 +POSTHOOK: Output: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +PREHOOK: query: explain +select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1_part_n0 + filterExpr: (empid > 20) (type: boolean) + Filter Operator + predicate: (empid > 20) (type: boolean) + Select Operator + expressions: deptno (type: int) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@depts_n00 +PREHOOK: Input: default@emps_n00 +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Input: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +POSTHOOK: query: select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts_n00 +POSTHOOK: Input: default@emps_n00 +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Input: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +20 +PREHOOK: query: drop materialized view mv1_part_n0 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0 +POSTHOOK: query: drop materialized view mv1_part_n0 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0 +PREHOOK: query: create materialized view mv1_part_n0 partitioned on (deptno) as +select empid, depts_n00.deptno as deptno from emps_n00 +join depts_n00 using (deptno) where emps_n00.deptno > 10 +group by empid, depts_n00.deptno +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts_n00 +PREHOOK: Input: default@emps_n00 +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0 +POSTHOOK: query: create materialized view mv1_part_n0 partitioned on (deptno) as +select empid, depts_n00.deptno as deptno from emps_n00 +join depts_n00 using (deptno) where emps_n00.deptno > 10 +group by empid, depts_n00.deptno +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts_n00 +POSTHOOK: Input: default@emps_n00 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0@deptno=20 +POSTHOOK: Lineage: mv1_part_n0 PARTITION(deptno=20).empid SIMPLE [(emps_n00)emps_n00.FieldSchema(name:empid, type:int, comment:null), ] +PREHOOK: query: analyze table mv1_part_n0 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Input: default@mv1_part_n0@deptno=20 +PREHOOK: Output: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1_part_n0 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Input: default@mv1_part_n0@deptno=20 +POSTHOOK: Output: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +PREHOOK: query: explain +select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1_part_n0 + filterExpr: (deptno > 20) (type: boolean) + Filter Operator + predicate: (deptno > 20) (type: boolean) + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@depts_n00 +PREHOOK: Input: default@emps_n00 +PREHOOK: Input: default@mv1_part_n0 +#### A masked pattern was here #### +POSTHOOK: query: select empid from emps_n00 +join depts_n00 using (deptno) where depts_n00.deptno > 20 +group by empid, depts_n00.deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts_n00 +POSTHOOK: Input: default@emps_n00 +POSTHOOK: Input: default@mv1_part_n0 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1_part_n0 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0 +POSTHOOK: query: drop materialized view mv1_part_n0 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0 +PREHOOK: query: create materialized view mv1_part_n0 partitioned on (deptno) as +select depts_n00.deptno as deptno, emps_n00.empid from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 10 +group by depts_n00.deptno, emps_n00.empid +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts_n00 +PREHOOK: Input: default@emps_n00 +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0 +POSTHOOK: query: create materialized view mv1_part_n0 partitioned on (deptno) as +select depts_n00.deptno as deptno, emps_n00.empid from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 10 +group by depts_n00.deptno, emps_n00.empid +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts_n00 +POSTHOOK: Input: default@emps_n00 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0@deptno=10 +POSTHOOK: Output: default@mv1_part_n0@deptno=20 +POSTHOOK: Lineage: mv1_part_n0 PARTITION(deptno=10).empid SIMPLE [(emps_n00)emps_n00.FieldSchema(name:empid, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n0 PARTITION(deptno=20).empid SIMPLE [(emps_n00)emps_n00.FieldSchema(name:empid, type:int, comment:null), ] +PREHOOK: query: analyze table mv1_part_n0 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Input: default@mv1_part_n0@deptno=10 +PREHOOK: Input: default@mv1_part_n0@deptno=20 +PREHOOK: Output: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0@deptno=10 +PREHOOK: Output: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1_part_n0 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Input: default@mv1_part_n0@deptno=10 +POSTHOOK: Input: default@mv1_part_n0@deptno=20 +POSTHOOK: Output: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0@deptno=10 +POSTHOOK: Output: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +PREHOOK: query: explain +select depts_n00.deptno from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 15 +group by depts_n00.deptno, emps_n00.empid +PREHOOK: type: QUERY +POSTHOOK: query: explain +select depts_n00.deptno from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 15 +group by depts_n00.deptno, emps_n00.empid +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.mv1_part_n0 + filterExpr: (deptno > 15) (type: boolean) + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: select depts_n00.deptno from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 15 +group by depts_n00.deptno, emps_n00.empid +PREHOOK: type: QUERY +PREHOOK: Input: default@depts_n00 +PREHOOK: Input: default@emps_n00 +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Input: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +POSTHOOK: query: select depts_n00.deptno from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 15 +group by depts_n00.deptno, emps_n00.empid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts_n00 +POSTHOOK: Input: default@emps_n00 +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Input: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +200 +PREHOOK: query: drop materialized view mv1_part_n0 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0 +POSTHOOK: query: drop materialized view mv1_part_n0 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0 +PREHOOK: query: create materialized view mv1_part_n0 partitioned on (deptno) as +select depts_n00.deptno as deptno, emps_n00.empid from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 10 +group by depts_n00.deptno, emps_n00.empid +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@depts_n00 +PREHOOK: Input: default@emps_n00 +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0 +POSTHOOK: query: create materialized view mv1_part_n0 partitioned on (deptno) as +select depts_n00.deptno as deptno, emps_n00.empid from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 10 +group by depts_n00.deptno, emps_n00.empid +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@depts_n00 +POSTHOOK: Input: default@emps_n00 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0@deptno=10 +POSTHOOK: Output: default@mv1_part_n0@deptno=20 +POSTHOOK: Lineage: mv1_part_n0 PARTITION(deptno=10).empid SIMPLE [(emps_n00)emps_n00.FieldSchema(name:empid, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n0 PARTITION(deptno=20).empid SIMPLE [(emps_n00)emps_n00.FieldSchema(name:empid, type:int, comment:null), ] +PREHOOK: query: analyze table mv1_part_n0 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Input: default@mv1_part_n0@deptno=10 +PREHOOK: Input: default@mv1_part_n0@deptno=20 +PREHOOK: Output: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0@deptno=10 +PREHOOK: Output: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1_part_n0 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Input: default@mv1_part_n0@deptno=10 +POSTHOOK: Input: default@mv1_part_n0@deptno=20 +POSTHOOK: Output: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0@deptno=10 +POSTHOOK: Output: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +PREHOOK: query: explain +select depts_n00.deptno from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 15 +group by depts_n00.deptno +PREHOOK: type: QUERY +POSTHOOK: query: explain +select depts_n00.deptno from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 15 +group by depts_n00.deptno +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1_part_n0 + filterExpr: (deptno > 15) (type: boolean) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: empid (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select depts_n00.deptno from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 15 +group by depts_n00.deptno +PREHOOK: type: QUERY +PREHOOK: Input: default@depts_n00 +PREHOOK: Input: default@emps_n00 +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Input: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +POSTHOOK: query: select depts_n00.deptno from depts_n00 +join emps_n00 using (deptno) where emps_n00.empid > 15 +group by depts_n00.deptno +POSTHOOK: type: QUERY +POSTHOOK: Input: default@depts_n00 +POSTHOOK: Input: default@emps_n00 +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Input: default@mv1_part_n0@deptno=20 +#### A masked pattern was here #### +200 +PREHOOK: query: drop materialized view mv1_part_n0 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0 +POSTHOOK: query: drop materialized view mv1_part_n0 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0 +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: create materialized view mv1_part_n0 partitioned on (deptno2) as +select depts_n00.name, dependents_n00.name as name2, emps_n00.deptno, depts_n00.deptno as deptno2, dependents_n00.empid +from depts_n00, dependents_n00, emps_n00 +where depts_n00.deptno > 10 +group by depts_n00.name, dependents_n00.name, emps_n00.deptno, depts_n00.deptno, dependents_n00.empid +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@dependents_n00 +PREHOOK: Input: default@depts_n00 +PREHOOK: Input: default@emps_n00 +PREHOOK: Output: database:default +PREHOOK: Output: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0 +POSTHOOK: query: create materialized view mv1_part_n0 partitioned on (deptno2) as +select depts_n00.name, dependents_n00.name as name2, emps_n00.deptno, depts_n00.deptno as deptno2, dependents_n00.empid +from depts_n00, dependents_n00, emps_n00 +where depts_n00.deptno > 10 +group by depts_n00.name, dependents_n00.name, emps_n00.deptno, depts_n00.deptno, dependents_n00.empid +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@dependents_n00 +POSTHOOK: Input: default@depts_n00 +POSTHOOK: Input: default@emps_n00 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0@deptno2=20 +POSTHOOK: Output: default@mv1_part_n0@deptno2=30 +POSTHOOK: Lineage: mv1_part_n0 PARTITION(deptno2=20).deptno SIMPLE [(emps_n00)emps_n00.FieldSchema(name:deptno, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n0 PARTITION(deptno2=20).empid SIMPLE [(dependents_n00)dependents_n00.FieldSchema(name:empid, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n0 PARTITION(deptno2=20).name SIMPLE [(depts_n00)depts_n00.FieldSchema(name:name, type:varchar(256), comment:null), ] +POSTHOOK: Lineage: mv1_part_n0 PARTITION(deptno2=20).name2 SIMPLE [(dependents_n00)dependents_n00.FieldSchema(name:name, type:varchar(256), comment:null), ] +POSTHOOK: Lineage: mv1_part_n0 PARTITION(deptno2=30).deptno SIMPLE [(emps_n00)emps_n00.FieldSchema(name:deptno, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n0 PARTITION(deptno2=30).empid SIMPLE [(dependents_n00)dependents_n00.FieldSchema(name:empid, type:int, comment:null), ] +POSTHOOK: Lineage: mv1_part_n0 PARTITION(deptno2=30).name SIMPLE [(depts_n00)depts_n00.FieldSchema(name:name, type:varchar(256), comment:null), ] +POSTHOOK: Lineage: mv1_part_n0 PARTITION(deptno2=30).name2 SIMPLE [(dependents_n00)dependents_n00.FieldSchema(name:name, type:varchar(256), comment:null), ] +PREHOOK: query: analyze table mv1_part_n0 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Input: default@mv1_part_n0@deptno2=20 +PREHOOK: Input: default@mv1_part_n0@deptno2=30 +PREHOOK: Output: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0@deptno2=20 +PREHOOK: Output: default@mv1_part_n0@deptno2=30 +#### A masked pattern was here #### +POSTHOOK: query: analyze table mv1_part_n0 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Input: default@mv1_part_n0@deptno2=20 +POSTHOOK: Input: default@mv1_part_n0@deptno2=30 +POSTHOOK: Output: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0@deptno2=20 +POSTHOOK: Output: default@mv1_part_n0@deptno2=30 +#### A masked pattern was here #### +PREHOOK: query: explain +select dependents_n00.empid +from depts_n00 +join dependents_n00 on (depts_n00.name = dependents_n00.name) +join emps_n00 on (emps_n00.deptno = depts_n00.deptno) +where depts_n00.deptno > 10 +group by dependents_n00.empid +PREHOOK: type: QUERY +POSTHOOK: query: explain +select dependents_n00.empid +from depts_n00 +join dependents_n00 on (depts_n00.name = dependents_n00.name) +join emps_n00 on (emps_n00.deptno = depts_n00.deptno) +where depts_n00.deptno > 10 +group by dependents_n00.empid +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.mv1_part_n0 + filterExpr: ((name = name2) and (deptno = empid)) (type: boolean) + Statistics: Num rows: 8 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((deptno = empid) and (name = name2)) (type: boolean) + Statistics: Num rows: 2 Data size: 390 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: deptno2 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 390 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select dependents_n00.empid +from depts_n00 +join dependents_n00 on (depts_n00.name = dependents_n00.name) +join emps_n00 on (emps_n00.deptno = depts_n00.deptno) +where depts_n00.deptno > 10 +group by dependents_n00.empid +PREHOOK: type: QUERY +PREHOOK: Input: default@dependents_n00 +PREHOOK: Input: default@depts_n00 +PREHOOK: Input: default@emps_n00 +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Input: default@mv1_part_n0@deptno2=20 +PREHOOK: Input: default@mv1_part_n0@deptno2=30 +#### A masked pattern was here #### +POSTHOOK: query: select dependents_n00.empid +from depts_n00 +join dependents_n00 on (depts_n00.name = dependents_n00.name) +join emps_n00 on (emps_n00.deptno = depts_n00.deptno) +where depts_n00.deptno > 10 +group by dependents_n00.empid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dependents_n00 +POSTHOOK: Input: default@depts_n00 +POSTHOOK: Input: default@emps_n00 +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Input: default@mv1_part_n0@deptno2=20 +POSTHOOK: Input: default@mv1_part_n0@deptno2=30 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view mv1_part_n0 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@mv1_part_n0 +PREHOOK: Output: default@mv1_part_n0 +POSTHOOK: query: drop materialized view mv1_part_n0 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@mv1_part_n0 +POSTHOOK: Output: default@mv1_part_n0