diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 73e6c21..0c23ab5 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1363,6 +1363,11 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "If the skew information is correctly stored in the metadata, hive.optimize.skewjoin.compiletime\n" + "would change the query plan to take care of it, and hive.optimize.skewjoin will be a no-op."), + // CTE + HIVE_CTE_MATERIALIZE_THRESHOLD("hive.optimize.cte.materialize.threshold", -1, + "If the number of references to a CTE clause exceeds this threshold, Hive will materialize it\n" + + "before executing the main query block. -1 will disable this feature."), + // Indexes HIVEOPTINDEXFILTER_COMPACT_MINSIZE("hive.optimize.index.filter.compact.minsize", (long) 5 * 1024 * 1024 * 1024, "Minimum size (in bytes) of the inputs on which a compact index is automatically used."), // 5G diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index fd2c924..8a928f2 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -99,6 +99,16 @@ minitez.query.files.shared=acid_globallimit.q,\ cross_product_check_1.q,\ cross_product_check_2.q,\ ctas.q,\ + cte_1.q,\ + cte_2.q,\ + cte_3.q,\ + cte_4.q,\ + cte_5.q,\ + cte_mat_1.q,\ + cte_mat_2.q,\ + cte_mat_3.q,\ + cte_mat_4.q,\ + cte_mat_5.q,\ custom_input_output_format.q,\ delete_all_non_partitioned.q,\ delete_all_partitioned.q,\ @@ -435,6 +445,16 @@ minitez.query.files=bucket_map_join_tez1.q,\ minillap.query.files=bucket_map_join_tez1.q,\ bucket_map_join_tez2.q,\ constprog_dpp.q,\ + cte_1.q,\ + cte_2.q,\ + cte_3.q,\ + cte_4.q,\ + cte_5.q,\ + cte_mat_1.q,\ + cte_mat_2.q,\ + cte_mat_3.q,\ + cte_mat_4.q,\ + cte_mat_5.q,\ dynamic_partition_pruning.q,\ dynamic_partition_pruning_2.q,\ hybridgrace_hashjoin_1.q,\ diff --git ql/src/java/org/apache/hadoop/hive/ql/Context.java ql/src/java/org/apache/hadoop/hive/ql/Context.java index 746456b..6f18c82 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -29,10 +29,9 @@ import java.util.Map; import java.util.Random; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import org.antlr.runtime.TokenRewriteStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; @@ -45,13 +44,15 @@ import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.lockmgr.HiveLock; -import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager; import org.apache.hadoop.hive.ql.lockmgr.HiveLockObj; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.LoadTableDesc; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Context for Semantic Analyzers. Usage: not reusable - construct a new one for @@ -108,6 +109,10 @@ private boolean needLockMgr; + private AtomicInteger sequencer = new AtomicInteger(); + + private final Map cteTables = new HashMap(); + // Keep track of the mapping from load table desc to the output and the lock private final Map loadTableOutputMap = new HashMap(); @@ -360,6 +365,28 @@ public void removeScratchDir() { fsScratchDirs.clear(); } + /** + * Remove any created directories for CTEs. + */ + public void removeMaterializedCTEs() { + // clean CTE tables + for (Table materializedTable : cteTables.values()) { + Path location = materializedTable.getDataLocation(); + try { + FileSystem fs = location.getFileSystem(conf); + if (fs.exists(location)) { + fs.delete(location, true); + LOG.info("Removed " + location + " for materialized " + materializedTable.getTableName()); + } + } catch (IOException e) { + // ignore + LOG.warn("Error removing " + location + " for materialized " + materializedTable.getTableName() + + ": " + StringUtils.stringifyException(e)); + } + } + cteTables.clear(); + } + private String nextPathId() { return Integer.toString(pathid++); } @@ -484,6 +511,7 @@ public void clear() throws IOException { LOG.info("Context clear error: " + StringUtils.stringifyException(e)); } } + removeMaterializedCTEs(); removeScratchDir(); originalTracker = null; setNeedLockMgr(false); @@ -717,6 +745,18 @@ public void setCboSucceeded(boolean cboSucceeded) { this.cboSucceeded = cboSucceeded; } + public Table getMaterializedTable(String cteName) { + return cteTables.get(cteName); + } + + public void addMaterializedTable(String cteName, Table table) { + cteTables.put(cteName, table); + } + + public AtomicInteger getSequencer() { + return sequencer; + } + public CompilationOpContext getOpContext() { return opContext; } diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 7147a9a..67bc4e1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -473,7 +473,7 @@ public void run() { sem.analyze(tree, ctx); hookCtx.update(sem); for (HiveSemanticAnalyzerHook hook : saHooks) { - hook.postAnalyze(hookCtx, sem.getRootTasks()); + hook.postAnalyze(hookCtx, sem.getAllRootTasks()); } } else { sem.analyze(tree, ctx); @@ -595,7 +595,7 @@ private String getExplainOutput(BaseSemanticAnalyzer sem, QueryPlan plan, ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(baos); try { - List> rootTasks = sem.getRootTasks(); + List> rootTasks = sem.getAllRootTasks(); task.getJSONPlan(ps, astTree, rootTasks, sem.getFetchTask(), false, true, true); ret = baos.toString(); } catch (Exception e) { diff --git ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java index 9132a21..4933b34 100644 --- ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java +++ ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java @@ -116,13 +116,12 @@ public QueryPlan(String queryString, BaseSemanticAnalyzer sem, Long startTime, S HiveOperation operation, Schema resultSchema) { this.queryString = queryString; - rootTasks = new ArrayList>(); - this.reducerTimeStatsPerJobList = new ArrayList(); - rootTasks.addAll(sem.getRootTasks()); + rootTasks = new ArrayList>(sem.getAllRootTasks()); + reducerTimeStatsPerJobList = new ArrayList(); fetchTask = sem.getFetchTask(); // Note that inputs and outputs can be changed when the query gets executed - inputs = sem.getInputs(); - outputs = sem.getOutputs(); + inputs = sem.getAllInputs(); + outputs = sem.getAllOutputs(); linfo = sem.getLineageInfo(); tableAccessInfo = sem.getTableAccessInfo(); columnAccessInfo = sem.getColumnAccessInfo(); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index be6ea63..a81eb18 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -18,18 +18,39 @@ package org.apache.hadoop.hive.ql.exec; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; +import static org.apache.commons.lang.StringUtils.join; +import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE; +import static org.apache.hadoop.util.StringUtils.stringifyException; + +import java.io.BufferedWriter; +import java.io.DataOutputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Serializable; +import java.io.Writer; +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.SQLException; +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; -import org.apache.hadoop.hive.ql.io.orc.OrcSerde; -import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -77,6 +98,8 @@ import org.apache.hadoop.hive.ql.io.RCFileInputFormat; import org.apache.hadoop.hive.ql.io.merge.MergeFileTask; import org.apache.hadoop.hive.ql.io.merge.MergeFileWork; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcSerde; import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; import org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateTask; import org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateWork; @@ -91,7 +114,6 @@ import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveMetaStoreChecker; -import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.InvalidTableException; import org.apache.hadoop.hive.ql.metadata.Partition; @@ -177,7 +199,6 @@ import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; -import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeSpec; import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; import org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe; @@ -194,42 +215,17 @@ import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatus; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.tools.HadoopArchives; import org.apache.hadoop.util.ToolRunner; import org.apache.hive.common.util.AnnotationUtils; import org.apache.hive.common.util.ReflectionUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.stringtemplate.v4.ST; -import java.io.BufferedWriter; -import java.io.DataOutputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.Serializable; -import java.io.Writer; -import java.net.URI; -import java.net.URISyntaxException; -import java.sql.SQLException; -import java.util.AbstractList; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; - -import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE; -import static org.apache.commons.lang.StringUtils.join; -import static org.apache.hadoop.util.StringUtils.stringifyException; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; /** * DDLTask implementation. @@ -3777,10 +3773,14 @@ private boolean updateModifiedParameters(Map params, HiveConf co return true; } + private void validateSerDe(String serdeName) throws HiveException { + validateSerDe(serdeName, conf); + } + /** * Check if the given serde is valid. */ - private void validateSerDe(String serdeName) throws HiveException { + public static void validateSerDe(String serdeName, HiveConf conf) throws HiveException { try { Deserializer d = ReflectionUtil.newInstance(conf.getClassByName(serdeName). @@ -3891,161 +3891,9 @@ private int switchDatabase(Hive db, SwitchDatabaseDesc switchDb) */ private int createTable(Hive db, CreateTableDesc crtTbl) throws HiveException { // create the table - Table tbl; - if (crtTbl.getDatabaseName() == null || (crtTbl.getTableName().contains("."))){ - tbl = db.newTable(crtTbl.getTableName()); - }else { - tbl = new Table(crtTbl.getDatabaseName(),crtTbl.getTableName()); - } - - if (crtTbl.getTblProps() != null) { - tbl.getTTable().getParameters().putAll(crtTbl.getTblProps()); - } - - if (crtTbl.getPartCols() != null) { - tbl.setPartCols(crtTbl.getPartCols()); - } - if (crtTbl.getNumBuckets() != -1) { - tbl.setNumBuckets(crtTbl.getNumBuckets()); - } - - if (crtTbl.getStorageHandler() != null) { - tbl.setProperty(META_TABLE_STORAGE, crtTbl.getStorageHandler()); - } - HiveStorageHandler storageHandler = tbl.getStorageHandler(); - - /* - * We use LazySimpleSerDe by default. - * - * If the user didn't specify a SerDe, and any of the columns are not simple - * types, we will have to use DynamicSerDe instead. - */ - if (crtTbl.getSerName() == null) { - if (storageHandler == null) { - LOG.info("Default to LazySimpleSerDe for table " + crtTbl.getTableName()); - tbl.setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); - } else { - String serDeClassName = storageHandler.getSerDeClass().getName(); - LOG.info("Use StorageHandler-supplied " + serDeClassName - + " for table " + crtTbl.getTableName()); - tbl.setSerializationLib(serDeClassName); - } - } else { - // let's validate that the serde exists - validateSerDe(crtTbl.getSerName()); - tbl.setSerializationLib(crtTbl.getSerName()); - } - - if (crtTbl.getFieldDelim() != null) { - tbl.setSerdeParam(serdeConstants.FIELD_DELIM, crtTbl.getFieldDelim()); - tbl.setSerdeParam(serdeConstants.SERIALIZATION_FORMAT, crtTbl.getFieldDelim()); - } - if (crtTbl.getFieldEscape() != null) { - tbl.setSerdeParam(serdeConstants.ESCAPE_CHAR, crtTbl.getFieldEscape()); - } - - if (crtTbl.getCollItemDelim() != null) { - tbl.setSerdeParam(serdeConstants.COLLECTION_DELIM, crtTbl.getCollItemDelim()); - } - if (crtTbl.getMapKeyDelim() != null) { - tbl.setSerdeParam(serdeConstants.MAPKEY_DELIM, crtTbl.getMapKeyDelim()); - } - if (crtTbl.getLineDelim() != null) { - tbl.setSerdeParam(serdeConstants.LINE_DELIM, crtTbl.getLineDelim()); - } - if (crtTbl.getNullFormat() != null) { - tbl.setSerdeParam(serdeConstants.SERIALIZATION_NULL_FORMAT, crtTbl.getNullFormat()); - } - if (crtTbl.getSerdeProps() != null) { - Iterator> iter = crtTbl.getSerdeProps().entrySet() - .iterator(); - while (iter.hasNext()) { - Entry m = iter.next(); - tbl.setSerdeParam(m.getKey(), m.getValue()); - } - } - - if (crtTbl.getCols() != null) { - tbl.setFields(crtTbl.getCols()); - } - if (crtTbl.getBucketCols() != null) { - tbl.setBucketCols(crtTbl.getBucketCols()); - } - if (crtTbl.getSortCols() != null) { - tbl.setSortCols(crtTbl.getSortCols()); - } - if (crtTbl.getComment() != null) { - tbl.setProperty("comment", crtTbl.getComment()); - } - if (crtTbl.getLocation() != null) { - tbl.setDataLocation(new Path(crtTbl.getLocation())); - } - - if (crtTbl.getSkewedColNames() != null) { - tbl.setSkewedColNames(crtTbl.getSkewedColNames()); - } - if (crtTbl.getSkewedColValues() != null) { - tbl.setSkewedColValues(crtTbl.getSkewedColValues()); - } - - tbl.getTTable().setTemporary(crtTbl.isTemporary()); - - tbl.setStoredAsSubDirectories(crtTbl.isStoredAsSubDirectories()); - - tbl.setInputFormatClass(crtTbl.getInputFormat()); - tbl.setOutputFormatClass(crtTbl.getOutputFormat()); - - // only persist input/output format to metadata when it is explicitly specified. - // Otherwise, load lazily via StorageHandler at query time. - if (crtTbl.getInputFormat() != null && !crtTbl.getInputFormat().isEmpty()) { - tbl.getTTable().getSd().setInputFormat(tbl.getInputFormatClass().getName()); - } - if (crtTbl.getOutputFormat() != null && !crtTbl.getOutputFormat().isEmpty()) { - tbl.getTTable().getSd().setOutputFormat(tbl.getOutputFormatClass().getName()); - } - - if (!Utilities.isDefaultNameNode(conf) && doesTableNeedLocation(tbl)) { - // If location is specified - ensure that it is a full qualified name - makeLocationQualified(tbl.getDbName(), tbl.getTTable().getSd(), tbl.getTableName()); - } - - if (crtTbl.isExternal()) { - tbl.setProperty("EXTERNAL", "TRUE"); - tbl.setTableType(TableType.EXTERNAL_TABLE); - } - - // If the sorted columns is a superset of bucketed columns, store this fact. - // It can be later used to - // optimize some group-by queries. Note that, the order does not matter as - // long as it in the first - // 'n' columns where 'n' is the length of the bucketed columns. - if ((tbl.getBucketCols() != null) && (tbl.getSortCols() != null)) { - List bucketCols = tbl.getBucketCols(); - List sortCols = tbl.getSortCols(); - - if ((sortCols.size() > 0) && (sortCols.size() >= bucketCols.size())) { - boolean found = true; - - Iterator iterBucketCols = bucketCols.iterator(); - while (iterBucketCols.hasNext()) { - String bucketCol = iterBucketCols.next(); - boolean colFound = false; - for (int i = 0; i < bucketCols.size(); i++) { - if (bucketCol.equals(sortCols.get(i).getCol())) { - colFound = true; - break; - } - } - if (colFound == false) { - found = false; - break; - } - } - if (found) { - tbl.setProperty("SORTBUCKETCOLSPREFIX", "TRUE"); - } - } - } + Table tbl = crtTbl.toTable(conf); + LOG.info("creating table " + tbl.getDbName() + "." + tbl.getTableName() + " on " + + tbl.getDataLocation()); // create the table if (crtTbl.getReplaceMode()){ @@ -4194,7 +4042,7 @@ private int createTableLike(Hive db, CreateTableLikeDesc crtTbl) throws Exceptio if (!Utilities.isDefaultNameNode(conf)) { // If location is specified - ensure that it is a full qualified name - makeLocationQualified(tbl.getDbName(), tbl.getTTable().getSd(), tbl.getTableName()); + makeLocationQualified(tbl.getDbName(), tbl.getTTable().getSd(), tbl.getTableName(), conf); } // create the table @@ -4409,8 +4257,8 @@ public String getName() { * @param name * Object name. */ - private void makeLocationQualified(String databaseName, StorageDescriptor sd, String name) - throws HiveException { + public static void makeLocationQualified(String databaseName, StorageDescriptor sd, + String name, HiveConf conf) throws HiveException { Path path = null; if (!sd.isSetLocation()) { @@ -4483,7 +4331,7 @@ private void makeLocationQualified(Database database) throws HiveException { } } - private static boolean doesTableNeedLocation(Table tbl) { + public static boolean doesTableNeedLocation(Table tbl) { // If we are ok with breaking compatibility of existing 3rd party StorageHandlers, // this method could be moved to the HiveStorageHandler interface. boolean retval = true; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java index 40c89cb..f2b15c5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java @@ -295,6 +295,22 @@ public boolean addDependentTask(Task dependent) { return ret; } + @SuppressWarnings("unchecked") + public static List> + findLeafs(List> rootTasks) { + final List> leafTasks = new ArrayList>(); + + NodeUtils.iterateTask(rootTasks, Task.class, new NodeUtils.Function() { + public void apply(Task task) { + List dependents = task.getDependentTasks(); + if (dependents == null || dependents.isEmpty()) { + leafTasks.add(task); + } + } + }); + return leafTasks; + } + /** * Remove the dependent task. * diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index a5217eb..5a72af1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -18,9 +18,16 @@ package org.apache.hadoop.hive.ql.metadata; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -57,15 +64,8 @@ import org.apache.hadoop.mapred.OutputFormat; import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hive.common.util.ReflectionUtil; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A Hive Table: is a fundamental unit of data in Hive that shares a common schema/DDL. @@ -94,6 +94,7 @@ private transient TableSpec tableSpec; + private transient boolean materializedTable; /** * Used only for serialization. @@ -338,6 +339,14 @@ public HiveStorageHandler getStorageHandler() { return outputFormatClass; } + public boolean isMaterializedTable() { + return materializedTable; + } + + public void setMaterializedTable(boolean materializedTable) { + this.materializedTable = materializedTable; + } + /** * Marker SemanticException, so that processing that allows for table validation failures * and appropriately handles them can recover from these types of SemanticExceptions diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 46bf04c..812af9a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -34,8 +34,6 @@ import java.util.Properties; import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; @@ -115,6 +113,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.mapred.InputFormat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; import com.google.common.collect.Interner; @@ -1742,7 +1742,8 @@ public static boolean isMergeRequired(List> mvTasks, HiveConf hco // no need of merging if the move is to a local file system MoveTask mvTask = (MoveTask) GenMapRedUtils.findMoveTask(mvTasks, fsOp); - if (mvTask != null && isInsertTable && hconf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER)) { + if (mvTask != null && isInsertTable && hconf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER) && + !fsOp.getConf().isMaterialization()) { GenMapRedUtils.addStatsTask(fsOp, mvTask, currTask, hconf); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index af1ee20..cb4a510 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -107,7 +107,7 @@ public static int HIVE_COLUMN_ORDER_DESC = 0; /** - * ReadEntitites that are passed to the hooks. + * ReadEntities that are passed to the hooks. */ protected HashSet inputs; /** @@ -1470,4 +1470,16 @@ protected Partition getPartition(Table table, Map partSpec, protected String toMessage(ErrorMsg message, Object detail) { return detail == null ? message.getMsg() : message.getMsg(detail.toString()); } + + public List> getAllRootTasks() { + return rootTasks; + } + + public HashSet getAllInputs() { + return inputs; + } + + public HashSet getAllOutputs() { + return outputs; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index e2d404b..e7dc08c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.parse; +import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.UndeclaredThrowableException; @@ -36,6 +37,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import org.antlr.runtime.ClassicToken; import org.antlr.runtime.tree.TreeVisitor; import org.antlr.runtime.tree.TreeVisitorAction; import org.apache.calcite.plan.RelOptCluster; @@ -96,6 +98,7 @@ import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Pair; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.conf.HiveConf.StrictChecks; @@ -110,6 +113,7 @@ import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException; @@ -185,6 +189,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.GroupByDesc; +import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.plan.SelectDesc; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; @@ -496,6 +501,54 @@ boolean continueJoinMerge() { } @Override + Table materializeCTE(String cteName, CTEClause cte) throws HiveException { + + ASTNode createTable = new ASTNode(new ClassicToken(HiveParser.TOK_CREATETABLE)); + + ASTNode tableName = new ASTNode(new ClassicToken(HiveParser.TOK_TABNAME)); + tableName.addChild(new ASTNode(new ClassicToken(HiveParser.Identifier, cteName))); + + ASTNode temporary = new ASTNode(new ClassicToken(HiveParser.KW_TEMPORARY, MATERIALIZATION_MARKER)); + + createTable.addChild(tableName); + createTable.addChild(temporary); + createTable.addChild(cte.cteNode); + + CalcitePlanner analyzer = new CalcitePlanner(conf); + analyzer.initCtx(ctx); + analyzer.init(false); + + // should share cte contexts + analyzer.aliasToCTEs.putAll(aliasToCTEs); + + HiveOperation operation = SessionState.get().getHiveOperation(); + try { + analyzer.analyzeInternal(createTable); + } finally { + SessionState.get().setCommandType(operation); + } + + Table table = analyzer.tableDesc.toTable(conf); + Path location = table.getDataLocation(); + try { + location.getFileSystem(conf).mkdirs(location); + } catch (IOException e) { + throw new HiveException(e); + } + table.setMaterializedTable(true); + + LOG.info(cteName + " will be materialized into " + location); + cte.table = table; + cte.source = analyzer; + + ctx.addMaterializedTable(cteName, table); + // For CalcitePlanner, store qualified name too + ctx.addMaterializedTable(table.getDbName() + "." + table.getTableName(), table); + + return table; + } + + @Override String fixCtasColumnName(String colName) { if (runCBO) { int lastDot = colName.lastIndexOf('.'); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java index e1e3eb2..e393be2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java @@ -75,7 +75,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { sem.validate(); ctx.setResFile(ctx.getLocalTmpPath()); - List> tasks = sem.getRootTasks(); + List> tasks = sem.getAllRootTasks(); if (tasks == null) { tasks = Collections.emptyList(); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezProcContext.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezProcContext.java index ec5ef0e..0c160ac 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezProcContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezProcContext.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; @@ -84,7 +85,7 @@ public Operator parentOfRoot; // sequence number is used to name vertices (e.g.: Map 1, Reduce 14, ...) - private int sequenceNumber = 0; + private AtomicInteger sequenceNumber; // tez task we're currently processing public TezTask currentTask; @@ -200,12 +201,12 @@ public GenTezProcContext(HiveConf conf, ParseContext parseContext, this.opMergeJoinWorkMap = new LinkedHashMap, MergeJoinWork>(); this.currentMergeJoinOperator = null; this.mapJoinToUnprocessedSmallTableReduceSinks = new HashMap>(); + this.sequenceNumber = parseContext.getContext().getSequencer(); rootTasks.add(currentTask); } - /** Not thread-safe. */ public int nextSequenceNumber() { - return ++sequenceNumber; + return sequenceNumber.incrementAndGet(); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index ba1945f..eceebe1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -221,6 +221,8 @@ import org.apache.hadoop.mapred.OutputFormat; import org.apache.hadoop.security.UserGroupInformation; +import com.google.common.collect.Sets; + /** * Implementation of the semantic analyzer. It generates the query plan. * There are other specific semantic analyzers for some hive operations such as @@ -239,6 +241,8 @@ private static final String VALUES_TMP_TABLE_NAME_PREFIX = "Values__Tmp__Table__"; + static final String MATERIALIZATION_MARKER = "$MATERIALIZATION"; + private HashMap opToPartPruner; private HashMap opToPartList; protected HashMap topOps; @@ -294,17 +298,24 @@ /* * Capture the CTE definitions in a Query. */ - private final Map aliasToCTEs; + final Map aliasToCTEs; + /* * Used to check recursive CTE invocations. Similar to viewsExpanded */ - private ArrayList ctesExpanded; + ArrayList ctesExpanded; + + /* + * Whether root tasks after materialized CTE linkage have been resolved + */ + boolean rootTasksResolved; + + CreateTableDesc tableDesc; /** Not thread-safe. */ final ASTSearcher astSearcher = new ASTSearcher(); protected AnalyzeRewriteContext analyzeRewrite; - private CreateTableDesc tableDesc; static class Phase1Ctx { String dest; @@ -341,7 +352,7 @@ public SemanticAnalyzer(HiveConf conf) throws SemanticException { HiveConf.ConfVars.HIVE_AUTOGEN_COLUMNALIAS_PREFIX_INCLUDEFUNCNAME); queryProperties = new QueryProperties(); opToPartToSkewedPruner = new HashMap>(); - aliasToCTEs = new HashMap(); + aliasToCTEs = new HashMap(); globalLimitCtx = new GlobalLimitCtx(); viewAliasToInput = new HashMap(); noscan = partialscan = false; @@ -961,7 +972,7 @@ private void processCTE(QB qb, ASTNode ctes) throws SemanticException { if ( aliasToCTEs.containsKey(qName)) { throw new SemanticException(ErrorMsg.AMBIGUOUS_TABLE_ALIAS.getMsg(cte.getChild(1))); } - aliasToCTEs.put(qName, cteQry); + aliasToCTEs.put(qName, new CTEClause(qName, cteQry)); } } @@ -976,7 +987,7 @@ private void processCTE(QB qb, ASTNode ctes) throws SemanticException { * they appear in when adding them to the aliasToCTEs map. * */ - private ASTNode findCTEFromName(QB qb, String cteName) { + private CTEClause findCTEFromName(QB qb, String cteName) { StringBuilder qId = new StringBuilder(); if (qb.getId() != null) { qId.append(qb.getId()); @@ -984,8 +995,9 @@ private ASTNode findCTEFromName(QB qb, String cteName) { while (qId.length() > 0) { String nm = qId + ":" + cteName; - if (aliasToCTEs.containsKey(nm)) { - return aliasToCTEs.get(nm); + CTEClause cte = aliasToCTEs.get(nm); + if (cte != null) { + return cte; } int lastIndex = qId.lastIndexOf(":"); lastIndex = lastIndex < 0 ? 0 : lastIndex; @@ -1005,14 +1017,180 @@ private ASTNode findCTEFromName(QB qb, String cteName) { * - trigger phase 1 on new QBExpr. * - update QB data structs: remove this as a table reference, move it to a SQ invocation. */ - private void addCTEAsSubQuery(QB qb, String cteName, String cteAlias) throws SemanticException { + private void addCTEAsSubQuery(QB qb, String cteName, String cteAlias) + throws SemanticException { cteAlias = cteAlias == null ? cteName : cteAlias; - ASTNode cteQryNode = findCTEFromName(qb, cteName); + CTEClause cte = findCTEFromName(qb, cteName); + ASTNode cteQryNode = cte.cteNode; QBExpr cteQBExpr = new QBExpr(cteAlias); doPhase1QBExpr(cteQryNode, cteQBExpr, qb.getId(), cteAlias); qb.rewriteCTEToSubq(cteAlias, cteName, cteQBExpr); } + private final CTEClause rootClause = new CTEClause(null, null); + + @Override + public List> getAllRootTasks() { + if (!rootTasksResolved) { + rootTasks = toRealRootTasks(rootClause.asExecutionOrder()); + rootTasksResolved = true; + } + return rootTasks; + } + + @Override + public HashSet getAllInputs() { + HashSet readEntities = new HashSet(getInputs()); + for (CTEClause cte : rootClause.asExecutionOrder()) { + if (cte.source != null) { + readEntities.addAll(cte.source.getInputs()); + } + } + return readEntities; + } + + @Override + public HashSet getAllOutputs() { + HashSet writeEntities = new HashSet(getOutputs()); + for (CTEClause cte : rootClause.asExecutionOrder()) { + if (cte.source != null) { + writeEntities.addAll(cte.source.getOutputs()); + } + } + return writeEntities; + } + + class CTEClause { + CTEClause(String alias, ASTNode cteNode) { + this.alias = alias; + this.cteNode = cteNode; + } + String alias; + ASTNode cteNode; + boolean materialize; + int reference; + QBExpr qbExpr; + List parents = new ArrayList(); + + // materialized + Table table; + SemanticAnalyzer source; + + List> getTasks() { + return source == null ? null : source.rootTasks; + } + + List asExecutionOrder() { + List execution = new ArrayList(); + asExecutionOrder(new HashSet(), execution); + return execution; + } + + void asExecutionOrder(Set visited, List execution) { + for (CTEClause parent : parents) { + if (visited.add(parent)) { + parent.asExecutionOrder(visited, execution); + } + } + execution.add(this); + } + + @Override + public String toString() { + return alias == null ? "" : alias; + } + } + + private List> toRealRootTasks(List execution) { + List> cteRoots = new ArrayList<>(); + List> cteLeafs = new ArrayList<>(); + List> curTopRoots = null; + List> curBottomLeafs = null; + for (int i = 0; i < execution.size(); i++) { + CTEClause current = execution.get(i); + if (current.parents.isEmpty() && curTopRoots != null) { + cteRoots.addAll(curTopRoots); + cteLeafs.addAll(curBottomLeafs); + curTopRoots = curBottomLeafs = null; + } + List> curTasks = current.getTasks(); + if (curTasks == null) { + continue; + } + if (curTopRoots == null) { + curTopRoots = curTasks; + } + if (curBottomLeafs != null) { + for (Task topLeafTask : curBottomLeafs) { + for (Task currentRootTask : curTasks) { + topLeafTask.addDependentTask(currentRootTask); + } + } + } + curBottomLeafs = Task.findLeafs(curTasks); + } + if (curTopRoots != null) { + cteRoots.addAll(curTopRoots); + cteLeafs.addAll(curBottomLeafs); + } + + if (cteRoots.isEmpty()) { + return rootTasks; + } + for (Task cteLeafTask : cteLeafs) { + for (Task mainRootTask : rootTasks) { + cteLeafTask.addDependentTask(mainRootTask); + } + } + return cteRoots; + } + + Table materializeCTE(String cteName, CTEClause cte) throws HiveException { + + ASTNode createTable = new ASTNode(new ClassicToken(HiveParser.TOK_CREATETABLE)); + + ASTNode tableName = new ASTNode(new ClassicToken(HiveParser.TOK_TABNAME)); + tableName.addChild(new ASTNode(new ClassicToken(HiveParser.Identifier, cteName))); + + ASTNode temporary = new ASTNode(new ClassicToken(HiveParser.KW_TEMPORARY, MATERIALIZATION_MARKER)); + + createTable.addChild(tableName); + createTable.addChild(temporary); + createTable.addChild(cte.cteNode); + + SemanticAnalyzer analyzer = new SemanticAnalyzer(conf); + analyzer.initCtx(ctx); + analyzer.init(false); + + // should share cte contexts + analyzer.aliasToCTEs.putAll(aliasToCTEs); + + HiveOperation operation = SessionState.get().getHiveOperation(); + try { + analyzer.analyzeInternal(createTable); + } finally { + SessionState.get().setCommandType(operation); + } + + Table table = analyzer.tableDesc.toTable(conf); + Path location = table.getDataLocation(); + try { + location.getFileSystem(conf).mkdirs(location); + } catch (IOException e) { + throw new HiveException(e); + } + table.setMaterializedTable(true); + + LOG.info(cteName + " will be materialized into " + location); + cte.table = table; + cte.source = analyzer; + + ctx.addMaterializedTable(cteName, table); + + return table; + } + + static boolean isJoinToken(ASTNode node) { if ((node.getToken().getType() == HiveParser.TOK_JOIN) || (node.getToken().getType() == HiveParser.TOK_CROSSJOIN) @@ -1538,141 +1716,217 @@ private void handleInsertStatementSpecPhase1(ASTNode ast, QBParseInfo qbp, Phase } } - private void getMetaData(QBExpr qbexpr, ReadEntity parentInput) - throws SemanticException { + public void getMaterializationMetadata(QB qb) throws SemanticException { + try { + gatherCTEReferences(qb, rootClause); + int threshold = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_CTE_MATERIALIZE_THRESHOLD); + for (CTEClause cte : Sets.newHashSet(aliasToCTEs.values())) { + if (threshold >= 0 && cte.reference >= threshold) { + cte.materialize = true; + } + } + } catch (HiveException e) { + // Has to use full name to make sure it does not conflict with + // org.apache.commons.lang.StringUtils + LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); + if (e instanceof SemanticException) { + throw (SemanticException)e; + } + throw new SemanticException(e.getMessage(), e); + } + } + + private void gatherCTEReferences(QBExpr qbexpr, CTEClause parent) throws HiveException { if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) { - getMetaData(qbexpr.getQB(), parentInput); + gatherCTEReferences(qbexpr.getQB(), parent); } else { - getMetaData(qbexpr.getQBExpr1(), parentInput); - getMetaData(qbexpr.getQBExpr2(), parentInput); + gatherCTEReferences(qbexpr.getQBExpr1(), parent); + gatherCTEReferences(qbexpr.getQBExpr2(), parent); } } - public Table getTable(TableScanOperator ts) { - return topToTable.get(ts); + // TODO: check view references, too + private void gatherCTEReferences(QB qb, CTEClause current) throws HiveException { + for (String alias : qb.getTabAliases()) { + String tabName = qb.getTabNameForAlias(alias); + String cteName = tabName.toLowerCase(); + + CTEClause cte = findCTEFromName(qb, cteName); + if (cte != null) { + if (ctesExpanded.contains(cteName)) { + throw new SemanticException("Recursive cte " + cteName + + " detected (cycle: " + StringUtils.join(ctesExpanded, " -> ") + + " -> " + cteName + ")."); + } + cte.reference++; + current.parents.add(cte); + if (cte.qbExpr != null) { + continue; + } + cte.qbExpr = new QBExpr(cteName); + doPhase1QBExpr(cte.cteNode, cte.qbExpr, qb.getId(), cteName); + + ctesExpanded.add(cteName); + gatherCTEReferences(cte.qbExpr, cte); + ctesExpanded.remove(ctesExpanded.size() - 1); + } + } + for (String alias : qb.getSubqAliases()) { + gatherCTEReferences(qb.getSubqForAlias(alias), current); + } } public void getMetaData(QB qb) throws SemanticException { - getMetaData(qb, null); + getMetaData(qb, false); } - @SuppressWarnings("nls") - public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException { + public void getMetaData(QB qb, boolean enableMaterialization) throws SemanticException { try { + if (enableMaterialization) { + getMaterializationMetadata(qb); + } + getMetaData(qb, null); + } catch (HiveException e) { + // Has to use full name to make sure it does not conflict with + // org.apache.commons.lang.StringUtils + LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); + if (e instanceof SemanticException) { + throw (SemanticException)e; + } + throw new SemanticException(e.getMessage(), e); + } + } - LOG.info("Get metadata for source tables"); + private void getMetaData(QBExpr qbexpr, ReadEntity parentInput) + throws HiveException { + if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) { + getMetaData(qbexpr.getQB(), parentInput); + } else { + getMetaData(qbexpr.getQBExpr1(), parentInput); + getMetaData(qbexpr.getQBExpr2(), parentInput); + } + } - // Go over the tables and populate the related structures. - // We have to materialize the table alias list since we might - // modify it in the middle for view rewrite. - List tabAliases = new ArrayList(qb.getTabAliases()); + @SuppressWarnings("nls") + private void getMetaData(QB qb, ReadEntity parentInput) + throws HiveException { + LOG.info("Get metadata for source tables"); - // Keep track of view alias to view name and read entity - // For eg: for a query like 'select * from V3', where V3 -> V2, V2 -> V1, V1 -> T - // keeps track of full view name and read entity corresponding to alias V3, V3:V2, V3:V2:V1. - // This is needed for tracking the dependencies for inputs, along with their parents. - Map> aliasToViewInfo = - new HashMap>(); + // Go over the tables and populate the related structures. + // We have to materialize the table alias list since we might + // modify it in the middle for view rewrite. + List tabAliases = new ArrayList(qb.getTabAliases()); - /* - * used to capture view to SQ conversions. This is used to check for - * recursive CTE invocations. - */ - Map sqAliasToCTEName = new HashMap(); + // Keep track of view alias to view name and read entity + // For eg: for a query like 'select * from V3', where V3 -> V2, V2 -> V1, V1 -> T + // keeps track of full view name and read entity corresponding to alias V3, V3:V2, V3:V2:V1. + // This is needed for tracking the dependencies for inputs, along with their parents. + Map> aliasToViewInfo = + new HashMap>(); - for (String alias : tabAliases) { - String tab_name = qb.getTabNameForAlias(alias); + /* + * used to capture view to SQ conversions. This is used to check for + * recursive CTE invocations. + */ + Map sqAliasToCTEName = new HashMap(); - // we first look for this alias from CTE, and then from catalog. - /* - * if this s a CTE reference: Add its AST as a SubQuery to this QB. - */ - ASTNode cteNode = findCTEFromName(qb, tab_name.toLowerCase()); - if (cteNode != null) { - String cte_name = tab_name.toLowerCase(); - if (ctesExpanded.contains(cte_name)) { - throw new SemanticException("Recursive cte " + tab_name + " detected (cycle: " - + StringUtils.join(ctesExpanded, " -> ") + " -> " + tab_name + ")."); + for (String alias : tabAliases) { + String tabName = qb.getTabNameForAlias(alias); + String cteName = tabName.toLowerCase(); + + Table tab = db.getTable(tabName, false); + if (tab == null || + tab.getDbName().equals(SessionState.get().getCurrentDatabase())) { + Table materializedTab = ctx.getMaterializedTable(cteName); + if (materializedTab == null) { + // we first look for this alias from CTE, and then from catalog. + CTEClause cte = findCTEFromName(qb, cteName); + if (cte != null) { + if (!cte.materialize) { + addCTEAsSubQuery(qb, cteName, alias); + sqAliasToCTEName.put(alias, cteName); + continue; + } + tab = materializeCTE(cteName, cte); } - addCTEAsSubQuery(qb, cte_name, alias); - sqAliasToCTEName.put(alias, cte_name); - continue; + } else { + tab = materializedTab; } + } - Table tab = db.getTable(tab_name, false); - if (tab == null) { - ASTNode src = qb.getParseInfo().getSrcForAlias(alias); - if (null != src) { - throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(src)); - } else { - throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(alias)); - } + if (tab == null) { + ASTNode src = qb.getParseInfo().getSrcForAlias(alias); + if (null != src) { + throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(src)); + } else { + throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(alias)); } + } - // Disallow INSERT INTO on bucketized tables - boolean isAcid = AcidUtils.isAcidTable(tab); - boolean isTableWrittenTo = qb.getParseInfo().isInsertIntoTable(tab.getDbName(), tab.getTableName()); - if (isTableWrittenTo && - tab.getNumBuckets() > 0 && !isAcid) { - throw new SemanticException(ErrorMsg.INSERT_INTO_BUCKETIZED_TABLE. - getMsg("Table: " + tab_name)); - } - // Disallow update and delete on non-acid tables - if ((updating() || deleting()) && !isAcid && isTableWrittenTo) { - //isTableWrittenTo: delete from acidTbl where a in (select id from nonAcidTable) - //so only assert this if we are actually writing to this table - // Whether we are using an acid compliant transaction manager has already been caught in - // UpdateDeleteSemanticAnalyzer, so if we are updating or deleting and getting nonAcid - // here, it means the table itself doesn't support it. - throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, tab_name); - } + // Disallow INSERT INTO on bucketized tables + boolean isAcid = AcidUtils.isAcidTable(tab); + boolean isTableWrittenTo = qb.getParseInfo().isInsertIntoTable(tab.getDbName(), tab.getTableName()); + if (isTableWrittenTo && + tab.getNumBuckets() > 0 && !isAcid) { + throw new SemanticException(ErrorMsg.INSERT_INTO_BUCKETIZED_TABLE. + getMsg("Table: " + tabName)); + } + // Disallow update and delete on non-acid tables + if ((updating() || deleting()) && !isAcid && isTableWrittenTo) { + //isTableWrittenTo: delete from acidTbl where a in (select id from nonAcidTable) + //so only assert this if we are actually writing to this table + // Whether we are using an acid compliant transaction manager has already been caught in + // UpdateDeleteSemanticAnalyzer, so if we are updating or deleting and getting nonAcid + // here, it means the table itself doesn't support it. + throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, tabName); + } - if (tab.isView()) { - if (qb.getParseInfo().isAnalyzeCommand()) { - throw new SemanticException(ErrorMsg.ANALYZE_VIEW.getMsg()); - } - String fullViewName = tab.getDbName() + "." + tab.getTableName(); - // Prevent view cycles - if (viewsExpanded.contains(fullViewName)) { - throw new SemanticException("Recursive view " + fullViewName + - " detected (cycle: " + StringUtils.join(viewsExpanded, " -> ") + - " -> " + fullViewName + ")."); - } - replaceViewReferenceWithDefinition(qb, tab, tab_name, alias); - // This is the last time we'll see the Table objects for views, so add it to the inputs - // now - ReadEntity viewInput = new ReadEntity(tab, parentInput); - viewInput = PlanUtils.addInput(inputs, viewInput); - aliasToViewInfo.put(alias, new ObjectPair(fullViewName, viewInput)); - viewAliasToInput.put(getAliasId(alias, qb), viewInput); - continue; - } + if (tab.isView()) { + if (qb.getParseInfo().isAnalyzeCommand()) { + throw new SemanticException(ErrorMsg.ANALYZE_VIEW.getMsg()); + } + String fullViewName = tab.getDbName() + "." + tab.getTableName(); + // Prevent view cycles + if (viewsExpanded.contains(fullViewName)) { + throw new SemanticException("Recursive view " + fullViewName + + " detected (cycle: " + StringUtils.join(viewsExpanded, " -> ") + + " -> " + fullViewName + ")."); + } + replaceViewReferenceWithDefinition(qb, tab, tabName, alias); + // This is the last time we'll see the Table objects for views, so add it to the inputs + // now + ReadEntity viewInput = new ReadEntity(tab, parentInput); + viewInput = PlanUtils.addInput(inputs, viewInput); + aliasToViewInfo.put(alias, new ObjectPair(fullViewName, viewInput)); + viewAliasToInput.put(getAliasId(alias, qb), viewInput); + continue; + } - if (!InputFormat.class.isAssignableFrom(tab.getInputFormatClass())) { - throw new SemanticException(generateErrorMessage( - qb.getParseInfo().getSrcForAlias(alias), - ErrorMsg.INVALID_INPUT_FORMAT_TYPE.getMsg())); - } + if (!InputFormat.class.isAssignableFrom(tab.getInputFormatClass())) { + throw new SemanticException(generateErrorMessage( + qb.getParseInfo().getSrcForAlias(alias), + ErrorMsg.INVALID_INPUT_FORMAT_TYPE.getMsg())); + } - qb.getMetaData().setSrcForAlias(alias, tab); + qb.getMetaData().setSrcForAlias(alias, tab); - if (qb.getParseInfo().isAnalyzeCommand()) { - // allow partial partition specification for nonscan since noscan is fast. - TableSpec ts = new TableSpec(db, conf, (ASTNode) ast.getChild(0), true, this.noscan); - if (ts.specType == SpecType.DYNAMIC_PARTITION) { // dynamic partitions - try { - ts.partitions = db.getPartitionsByNames(ts.tableHandle, ts.partSpec); - } catch (HiveException e) { - throw new SemanticException(generateErrorMessage( - qb.getParseInfo().getSrcForAlias(alias), - "Cannot get partitions for " + ts.partSpec), e); - } + if (qb.getParseInfo().isAnalyzeCommand()) { + // allow partial partition specification for nonscan since noscan is fast. + TableSpec ts = new TableSpec(db, conf, (ASTNode) ast.getChild(0), true, this.noscan); + if (ts.specType == SpecType.DYNAMIC_PARTITION) { // dynamic partitions + try { + ts.partitions = db.getPartitionsByNames(ts.tableHandle, ts.partSpec); + } catch (HiveException e) { + throw new SemanticException(generateErrorMessage( + qb.getParseInfo().getSrcForAlias(alias), + "Cannot get partitions for " + ts.partSpec), e); } - // validate partial scan command - QBParseInfo qbpi = qb.getParseInfo(); - if (qbpi.isPartialScanAnalyzeCommand()) { - Class inputFormatClass = null; - switch (ts.specType) { + } + // validate partial scan command + QBParseInfo qbpi = qb.getParseInfo(); + if (qbpi.isPartialScanAnalyzeCommand()) { + Class inputFormatClass = null; + switch (ts.specType) { case TABLE_ONLY: case DYNAMIC_PARTITION: inputFormatClass = ts.tableHandle.getInputFormatClass(); @@ -1682,55 +1936,55 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException break; default: assert false; - } - // throw a HiveException for formats other than rcfile or orcfile. - if (!(inputFormatClass.equals(RCFileInputFormat.class) || inputFormatClass - .equals(OrcInputFormat.class))) { - throw new SemanticException(ErrorMsg.ANALYZE_TABLE_PARTIALSCAN_NON_RCFILE.getMsg()); - } } - - tab.setTableSpec(ts); - qb.getParseInfo().addTableSpec(alias, ts); + // throw a HiveException for formats other than rcfile or orcfile. + if (!(inputFormatClass.equals(RCFileInputFormat.class) || inputFormatClass + .equals(OrcInputFormat.class))) { + throw new SemanticException(ErrorMsg.ANALYZE_TABLE_PARTIALSCAN_NON_RCFILE.getMsg()); + } } - ReadEntity parentViewInfo = PlanUtils.getParentViewInfo(getAliasId(alias, qb), viewAliasToInput); - PlanUtils.addInput(inputs, - new ReadEntity(tab, parentViewInfo, parentViewInfo == null)); + tab.setTableSpec(ts); + qb.getParseInfo().addTableSpec(alias, ts); } - LOG.info("Get metadata for subqueries"); - // Go over the subqueries and getMetaData for these - for (String alias : qb.getSubqAliases()) { - boolean wasView = aliasToViewInfo.containsKey(alias); - boolean wasCTE = sqAliasToCTEName.containsKey(alias); - ReadEntity newParentInput = null; - if (wasView) { - viewsExpanded.add(aliasToViewInfo.get(alias).getFirst()); - newParentInput = aliasToViewInfo.get(alias).getSecond(); - } else if (wasCTE) { - ctesExpanded.add(sqAliasToCTEName.get(alias)); - } - QBExpr qbexpr = qb.getSubqForAlias(alias); - getMetaData(qbexpr, newParentInput); - if (wasView) { - viewsExpanded.remove(viewsExpanded.size() - 1); - } else if (wasCTE) { - ctesExpanded.remove(ctesExpanded.size() - 1); - } + ReadEntity parentViewInfo = PlanUtils.getParentViewInfo(getAliasId(alias, qb), viewAliasToInput); + PlanUtils.addInput(inputs, + new ReadEntity(tab, parentViewInfo, parentViewInfo == null)); + } + + LOG.info("Get metadata for subqueries"); + // Go over the subqueries and getMetaData for these + for (String alias : qb.getSubqAliases()) { + boolean wasView = aliasToViewInfo.containsKey(alias); + boolean wasCTE = sqAliasToCTEName.containsKey(alias); + ReadEntity newParentInput = null; + if (wasView) { + viewsExpanded.add(aliasToViewInfo.get(alias).getFirst()); + newParentInput = aliasToViewInfo.get(alias).getSecond(); + } else if (wasCTE) { + ctesExpanded.add(sqAliasToCTEName.get(alias)); + } + QBExpr qbexpr = qb.getSubqForAlias(alias); + getMetaData(qbexpr, newParentInput); + if (wasView) { + viewsExpanded.remove(viewsExpanded.size() - 1); + } else if (wasCTE) { + ctesExpanded.remove(ctesExpanded.size() - 1); } + } - RowFormatParams rowFormatParams = new RowFormatParams(); - StorageFormat storageFormat = new StorageFormat(conf); + RowFormatParams rowFormatParams = new RowFormatParams(); + StorageFormat storageFormat = new StorageFormat(conf); - LOG.info("Get metadata for destination tables"); - // Go over all the destination structures and populate the related - // metadata - QBParseInfo qbp = qb.getParseInfo(); + LOG.info("Get metadata for destination tables"); + // Go over all the destination structures and populate the related + // metadata + QBParseInfo qbp = qb.getParseInfo(); - for (String name : qbp.getClauseNamesForDest()) { - ASTNode ast = qbp.getDestForClause(name); - switch (ast.getToken().getType()) { + for (String name : qbp.getClauseNamesForDest()) { + ASTNode ast = qbp.getDestForClause(name); + switch (ast.getToken().getType()) { case HiveParser.TOK_TAB: { TableSpec ts = new TableSpec(db, conf, ast); if (ts.tableHandle.isView()) { @@ -1853,13 +2107,7 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException default: throw new SemanticException(generateErrorMessage(ast, "Unknown Token Type " + ast.getToken().getType())); - } } - } catch (HiveException e) { - // Has to use full name to make sure it does not conflict with - // org.apache.commons.lang.StringUtils - LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); - throw new SemanticException(e.getMessage(), e); } } @@ -6188,6 +6436,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) Table dest_tab = null; // destination table if any boolean destTableIsAcid = false; // should the destination table be written to using ACID boolean destTableIsTemporary = false; + boolean destTableIsMaterialization = false; Partition dest_part = null;// destination partition if any Path queryTmpdir = null; // the intermediate destination directory Path dest_path = null; // the final destination directory @@ -6447,6 +6696,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) if (tblDesc != null) { field_schemas = new ArrayList(); destTableIsTemporary = tblDesc.isTemporary(); + destTableIsMaterialization = tblDesc.isMaterialization(); } boolean first = true; @@ -6604,6 +6854,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) } fileSinkDesc.setTemporary(destTableIsTemporary); + fileSinkDesc.setMaterialization(destTableIsMaterialization); /* Set List Bucketing context. */ if (lbCtx != null) { @@ -6621,7 +6872,8 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // can be changed in the optimizer but the key should not be changed // it should be the same as the MoveWork's sourceDir. fileSinkDesc.setStatsAggPrefix(fileSinkDesc.getDirName().toString()); - if (HiveConf.getVar(conf, HIVESTATSDBCLASS).equalsIgnoreCase(StatDB.fs.name())) { + if (!destTableIsMaterialization && + HiveConf.getVar(conf, HIVESTATSDBCLASS).equalsIgnoreCase(StatDB.fs.name())) { String statsTmpLoc = ctx.getExtTmpPathRelTo(queryTmpdir).toString(); fileSinkDesc.setStatsTmpDir(statsTmpLoc); LOG.debug("Set stats collection dir : " + statsTmpLoc); @@ -10036,7 +10288,8 @@ else if(ast.getChild(0).getType() == HiveParser.TOK_FALSE) { LOG.info("Completed phase 1 of Semantic Analysis"); // 5. Resolve Parse Tree - getMetaData(qb); + // Materialization is allowed if it is not a view definition + getMetaData(qb, createVwDesc == null); LOG.info("Completed getting MetaData in Semantic Analysis"); plannerCtx.setParseTreeAttr(child, ctx_1); @@ -10744,6 +10997,7 @@ ASTNode analyzeCreateTable( boolean ifNotExists = false; boolean isExt = false; boolean isTemporary = false; + boolean isMaterialization = false; ASTNode selectStmt = null; final int CREATE_TABLE = 0; // regular CREATE TABLE final int CTLT = 1; // CREATE TABLE LIKE ... (CTLT) @@ -10783,6 +11037,7 @@ ASTNode analyzeCreateTable( break; case HiveParser.KW_TEMPORARY: isTemporary = true; + isMaterialization = MATERIALIZATION_MARKER.equals(child.getText()); break; case HiveParser.TOK_LIKETABLE: if (child.getChildCount() > 0) { @@ -10972,17 +11227,19 @@ ASTNode analyzeCreateTable( case CTAS: // create table as select if (isTemporary) { - String dbName = qualifiedTabName[0]; - String tblName = qualifiedTabName[1]; - SessionState ss = SessionState.get(); - if (ss == null) { - throw new SemanticException("No current SessionState, cannot create temporary table " - + dbName + "." + tblName); - } - Map tables = SessionHiveMetaStoreClient.getTempTablesForDatabase(dbName); - if (tables != null && tables.containsKey(tblName) && !ctx.getExplain()) { - throw new SemanticException("Temporary table " + dbName + "." + tblName - + " already exists"); + if (!ctx.getExplain() && !isMaterialization) { + String dbName = qualifiedTabName[0]; + String tblName = qualifiedTabName[1]; + SessionState ss = SessionState.get(); + if (ss == null) { + throw new SemanticException("No current SessionState, cannot create temporary table " + + dbName + "." + tblName); + } + Map tables = SessionHiveMetaStoreClient.getTempTablesForDatabase(dbName); + if (tables != null && tables.containsKey(tblName)) { + throw new SemanticException("Temporary table " + dbName + "." + tblName + + " already exists"); + } } } else { // Verify that the table does not already exist @@ -11031,6 +11288,7 @@ ASTNode analyzeCreateTable( storageFormat.getOutputFormat(), location, storageFormat.getSerde(), storageFormat.getStorageHandler(), storageFormat.getSerdeProps(), tblProps, ifNotExists, skewedColNames, skewedValues); + tableDesc.setMaterialization(isMaterialization); tableDesc.setStoredAsSubDirectories(storedAsDirs); tableDesc.setNullFormat(rowFormatParams.nullFormat); qb.setTableDesc(tableDesc); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index 89897d7..fc555ca 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -228,7 +228,7 @@ public void compile(final ParseContext pCtx, final List> skewedColValues; boolean isStoredAsSubDirectories = false; boolean isTemporary = false; + private boolean isMaterialization = false; private boolean replaceMode = false; public CreateTableDesc() { @@ -553,6 +561,21 @@ public void setTemporary(boolean isTemporary) { } /** + * @return the isMaterialization + */ + @Explain(displayName = "isMaterialization", displayOnlyOnTrue = true) + public boolean isMaterialization() { + return isMaterialization; + } + + /** + * @param isMaterialization table is a materialization or not. + */ + public void setMaterialization(boolean isMaterialization) { + this.isMaterialization = isMaterialization; + } + + /** * @param replaceMode Determine if this CreateTable should behave like a replace-into alter instead */ public void setReplaceMode(boolean replaceMode) { @@ -565,4 +588,170 @@ public void setReplaceMode(boolean replaceMode) { public boolean getReplaceMode() { return replaceMode; } + + public Table toTable(HiveConf conf) throws HiveException { + String databaseName = getDatabaseName(); + String tableName = getTableName(); + + if (databaseName == null || tableName.contains(".")) { + String[] names = Utilities.getDbTableName(tableName); + databaseName = names[0]; + tableName = names[1]; + } + + Table tbl = new Table(databaseName, tableName); + + if (getTblProps() != null) { + tbl.getTTable().getParameters().putAll(getTblProps()); + } + + if (getPartCols() != null) { + tbl.setPartCols(getPartCols()); + } + if (getNumBuckets() != -1) { + tbl.setNumBuckets(getNumBuckets()); + } + + if (getStorageHandler() != null) { + tbl.setProperty( + org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, + getStorageHandler()); + } + HiveStorageHandler storageHandler = tbl.getStorageHandler(); + + /* + * We use LazySimpleSerDe by default. + * + * If the user didn't specify a SerDe, and any of the columns are not simple + * types, we will have to use DynamicSerDe instead. + */ + if (getSerName() == null) { + if (storageHandler == null) { + LOG.info("Default to LazySimpleSerDe for table " + tableName); + tbl.setSerializationLib(LazySimpleSerDe.class.getName()); + } else { + String serDeClassName = storageHandler.getSerDeClass().getName(); + LOG.info("Use StorageHandler-supplied " + serDeClassName + + " for table " + tableName); + tbl.setSerializationLib(serDeClassName); + } + } else { + // let's validate that the serde exists + DDLTask.validateSerDe(getSerName(), conf); + tbl.setSerializationLib(getSerName()); + } + + if (getFieldDelim() != null) { + tbl.setSerdeParam(serdeConstants.FIELD_DELIM, getFieldDelim()); + tbl.setSerdeParam(serdeConstants.SERIALIZATION_FORMAT, getFieldDelim()); + } + if (getFieldEscape() != null) { + tbl.setSerdeParam(serdeConstants.ESCAPE_CHAR, getFieldEscape()); + } + + if (getCollItemDelim() != null) { + tbl.setSerdeParam(serdeConstants.COLLECTION_DELIM, getCollItemDelim()); + } + if (getMapKeyDelim() != null) { + tbl.setSerdeParam(serdeConstants.MAPKEY_DELIM, getMapKeyDelim()); + } + if (getLineDelim() != null) { + tbl.setSerdeParam(serdeConstants.LINE_DELIM, getLineDelim()); + } + if (getNullFormat() != null) { + tbl.setSerdeParam(serdeConstants.SERIALIZATION_NULL_FORMAT, getNullFormat()); + } + if (getSerdeProps() != null) { + Iterator> iter = getSerdeProps().entrySet() + .iterator(); + while (iter.hasNext()) { + Map.Entry m = iter.next(); + tbl.setSerdeParam(m.getKey(), m.getValue()); + } + } + + if (getCols() != null) { + tbl.setFields(getCols()); + } + if (getBucketCols() != null) { + tbl.setBucketCols(getBucketCols()); + } + if (getSortCols() != null) { + tbl.setSortCols(getSortCols()); + } + if (getComment() != null) { + tbl.setProperty("comment", getComment()); + } + if (getLocation() != null) { + tbl.setDataLocation(new Path(getLocation())); + } + + if (getSkewedColNames() != null) { + tbl.setSkewedColNames(getSkewedColNames()); + } + if (getSkewedColValues() != null) { + tbl.setSkewedColValues(getSkewedColValues()); + } + + tbl.getTTable().setTemporary(isTemporary()); + + tbl.setStoredAsSubDirectories(isStoredAsSubDirectories()); + + tbl.setInputFormatClass(getInputFormat()); + tbl.setOutputFormatClass(getOutputFormat()); + + // only persist input/output format to metadata when it is explicitly specified. + // Otherwise, load lazily via StorageHandler at query time. + if (getInputFormat() != null && !getInputFormat().isEmpty()) { + tbl.getTTable().getSd().setInputFormat(tbl.getInputFormatClass().getName()); + } + if (getOutputFormat() != null && !getOutputFormat().isEmpty()) { + tbl.getTTable().getSd().setOutputFormat(tbl.getOutputFormatClass().getName()); + } + + if (!Utilities.isDefaultNameNode(conf) && DDLTask.doesTableNeedLocation(tbl)) { + // If location is specified - ensure that it is a full qualified name + DDLTask.makeLocationQualified(tbl.getDbName(), tbl.getTTable().getSd(), tableName, conf); + } + + if (isExternal()) { + tbl.setProperty("EXTERNAL", "TRUE"); + tbl.setTableType(TableType.EXTERNAL_TABLE); + } + + // If the sorted columns is a superset of bucketed columns, store this fact. + // It can be later used to + // optimize some group-by queries. Note that, the order does not matter as + // long as it in the first + // 'n' columns where 'n' is the length of the bucketed columns. + if ((tbl.getBucketCols() != null) && (tbl.getSortCols() != null)) { + List bucketCols = tbl.getBucketCols(); + List sortCols = tbl.getSortCols(); + + if ((sortCols.size() > 0) && (sortCols.size() >= bucketCols.size())) { + boolean found = true; + + Iterator iterBucketCols = bucketCols.iterator(); + while (iterBucketCols.hasNext()) { + String bucketCol = iterBucketCols.next(); + boolean colFound = false; + for (int i = 0; i < bucketCols.size(); i++) { + if (bucketCol.equals(sortCols.get(i).getCol())) { + colFound = true; + break; + } + } + if (colFound == false) { + found = false; + break; + } + } + if (found) { + tbl.setProperty("SORTBUCKETCOLSPREFIX", "TRUE"); + } + } + } + return tbl; + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java index cc462be..07fd2dc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java @@ -51,6 +51,7 @@ private String compressType; private boolean multiFileSpray; private boolean temporary; + private boolean materialization; // Whether the files output by this FileSink can be merged, e.g. if they are to be put into a // bucketed or sorted table/partition they cannot be merged. private boolean canBeMerged; @@ -241,6 +242,14 @@ public void setTemporary(boolean temporary) { this.temporary = temporary; } + public boolean isMaterialization() { + return materialization; + } + + public void setMaterialization(boolean materialization) { + this.materialization = materialization; + } + public boolean canBeMerged() { return canBeMerged; diff --git ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index ea506fc..9d139ba 100644 --- ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -18,12 +18,16 @@ package org.apache.hadoop.hive.ql.stats; -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; -import com.google.common.math.LongMath; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -44,7 +48,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.optimizer.stats.annotation.StatsRulesProcFactory; import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ColStatistics; @@ -93,16 +96,12 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hive.common.util.AnnotationUtils; import org.apache.tez.mapreduce.hadoop.MRJobConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.google.common.math.LongMath; public class StatsUtils { @@ -745,6 +744,10 @@ public static ColStatistics getColStatistics(ColumnStatisticsObj cso, String tab */ public static List getTableColumnStats( Table table, List schema, List neededColumns) { + if (table.isMaterializedTable()) { + LOG.debug("Materialized table does not contain table statistics"); + return null; + } String dbName = table.getDbName(); String tabName = table.getTableName(); List neededColsInTable = processNeededColumns(schema, neededColumns); diff --git ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java index be51edc..fc42aaa 100644 --- ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java +++ ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CompilationOpContext; +import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; @@ -42,6 +43,7 @@ import org.apache.hadoop.hive.ql.plan.ReduceWork; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.TezWork; +import org.apache.hadoop.hive.ql.session.SessionState; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -64,9 +66,17 @@ @SuppressWarnings("unchecked") @Before public void setUp() throws Exception { + // Init conf + final HiveConf conf = new HiveConf(SemanticAnalyzer.class); + SessionState.start(conf); + + // Init parse context + final ParseContext pctx = new ParseContext(); + pctx.setContext(new Context(conf)); + ctx = new GenTezProcContext( - new HiveConf(), - new ParseContext(), + conf, + pctx, Collections.EMPTY_LIST, new ArrayList>(), Collections.EMPTY_SET, diff --git ql/src/test/queries/clientpositive/cte_3.q ql/src/test/queries/clientpositive/cte_3.q new file mode 100644 index 0000000..aa8adbc --- /dev/null +++ ql/src/test/queries/clientpositive/cte_3.q @@ -0,0 +1,31 @@ +set hive.mapred.mode=nonstrict; +set hive.optimize.cte.materialize.threshold=1; + +explain +with q1 as ( select key from src where key = '5') +select * +from q1 +; + +with q1 as ( select key from src where key = '5') +select * +from q1 +; + +-- in subquery +explain +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a; + +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a; + +-- chaining +explain +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a; + +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a; diff --git ql/src/test/queries/clientpositive/cte_4.q ql/src/test/queries/clientpositive/cte_4.q new file mode 100644 index 0000000..0455a6a --- /dev/null +++ ql/src/test/queries/clientpositive/cte_4.q @@ -0,0 +1,56 @@ +set hive.mapred.mode=nonstrict; +set hive.optimize.cte.materialize.threshold=1; + +-- union test +with q1 as (select * from src where key= '5'), +q2 as (select * from src s2 where key = '4') +select * from q1 union all select * from q2 +; + +-- insert test +create table s1 like src; +with q1 as ( select key, value from src where key = '5') +from q1 +insert overwrite table s1 +select * +; +select * from s1; +drop table s1; + +-- from style +with q1 as (select * from src where key= '5') +from q1 +select * +; + +-- ctas +create table s2 as +with q1 as ( select key from src where key = '4') +select * from q1 +; + +select * from s2; +drop table s2; + +-- view test +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +; + +select * from v1; + +drop view v1; + + +-- view test, name collision +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +; + +with q1 as ( select key from src where key = '4') +select * from v1 +; + +drop view v1; diff --git ql/src/test/queries/clientpositive/cte_5.q ql/src/test/queries/clientpositive/cte_5.q new file mode 100644 index 0000000..8968688 --- /dev/null +++ ql/src/test/queries/clientpositive/cte_5.q @@ -0,0 +1,23 @@ +set hive.mapred.mode=nonstrict; +set hive.optimize.cte.materialize.threshold=-1; + +create database mydb; +use mydb; +create table q1 (colnum int, colstring string); +insert into q1 values (5, 'A'); + +use default; + +show tables in mydb; +show tables; + +explain +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key; + +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key; diff --git ql/src/test/queries/clientpositive/cte_mat_1.q ql/src/test/queries/clientpositive/cte_mat_1.q new file mode 100644 index 0000000..2afb960 --- /dev/null +++ ql/src/test/queries/clientpositive/cte_mat_1.q @@ -0,0 +1,8 @@ +set hive.mapred.mode=nonstrict; +set hive.optimize.cte.materialize.threshold=-1; + +explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key; diff --git ql/src/test/queries/clientpositive/cte_mat_2.q ql/src/test/queries/clientpositive/cte_mat_2.q new file mode 100644 index 0000000..adcd087 --- /dev/null +++ ql/src/test/queries/clientpositive/cte_mat_2.q @@ -0,0 +1,8 @@ +set hive.mapred.mode=nonstrict; +set hive.optimize.cte.materialize.threshold=3; + +explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key; diff --git ql/src/test/queries/clientpositive/cte_mat_3.q ql/src/test/queries/clientpositive/cte_mat_3.q new file mode 100644 index 0000000..650cc24 --- /dev/null +++ ql/src/test/queries/clientpositive/cte_mat_3.q @@ -0,0 +1,8 @@ +set hive.mapred.mode=nonstrict; +set hive.optimize.cte.materialize.threshold=2; + +explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key; diff --git ql/src/test/queries/clientpositive/cte_mat_4.q ql/src/test/queries/clientpositive/cte_mat_4.q new file mode 100644 index 0000000..2d75963 --- /dev/null +++ ql/src/test/queries/clientpositive/cte_mat_4.q @@ -0,0 +1,39 @@ +set hive.mapred.mode=nonstrict; +set hive.optimize.cte.materialize.threshold=2; + +create temporary table q1 (a int, b string); +insert into q1 values (1, 'A'); + +show tables; + +explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key; + +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key; + +show tables; + +select * from q1; + +drop table q1; + +show tables; + +explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key; + +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key; + +show tables; diff --git ql/src/test/queries/clientpositive/cte_mat_5.q ql/src/test/queries/clientpositive/cte_mat_5.q new file mode 100644 index 0000000..fd0aeda --- /dev/null +++ ql/src/test/queries/clientpositive/cte_mat_5.q @@ -0,0 +1,23 @@ +set hive.mapred.mode=nonstrict; +set hive.optimize.cte.materialize.threshold=1; + +create database mydb; +use mydb; +create table q1 (colnum int, colstring string); +insert into q1 values (5, 'A'); + +use default; + +show tables in mydb; +show tables; + +explain +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key; + +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key; diff --git ql/src/test/results/clientnegative/analyze1.q.out ql/src/test/results/clientnegative/analyze1.q.out index 589a6ee..3230b02 100644 --- ql/src/test/results/clientnegative/analyze1.q.out +++ ql/src/test/results/clientnegative/analyze1.q.out @@ -1 +1 @@ -FAILED: SemanticException Partition spec {key=null} contains non-partition columns +FAILED: ValidationFailureSemanticException Partition spec {key=null} contains non-partition columns diff --git ql/src/test/results/clientnegative/dyn_part1.q.out ql/src/test/results/clientnegative/dyn_part1.q.out index 62d01fb..4c8e171 100644 --- ql/src/test/results/clientnegative/dyn_part1.q.out +++ ql/src/test/results/clientnegative/dyn_part1.q.out @@ -6,4 +6,4 @@ POSTHOOK: query: create table dynamic_partition (key string) partitioned by (val POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@dynamic_partition -FAILED: SemanticException Partition spec {hr=null} contains non-partition columns +FAILED: ValidationFailureSemanticException Partition spec {hr=null} contains non-partition columns diff --git ql/src/test/results/clientpositive/alter_view_as_select.q.out ql/src/test/results/clientpositive/alter_view_as_select.q.out index c89c0dc..3666221 100644 --- ql/src/test/results/clientpositive/alter_view_as_select.q.out +++ ql/src/test/results/clientpositive/alter_view_as_select.q.out @@ -57,7 +57,6 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:tv POSTHOOK: Output: tv@testView -POSTHOOK: Output: tv@testview PREHOOK: query: DESCRIBE FORMATTED tv.testView PREHOOK: type: DESCTABLE PREHOOK: Input: tv@testview @@ -106,7 +105,6 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:tv POSTHOOK: Output: tv@testView -POSTHOOK: Output: tv@testview PREHOOK: query: DESCRIBE FORMATTED tv.testView PREHOOK: type: DESCTABLE PREHOOK: Input: tv@testview diff --git ql/src/test/results/clientpositive/cte_3.q.out ql/src/test/results/clientpositive/cte_3.q.out new file mode 100644 index 0000000..0fe0865 --- /dev/null +++ ql/src/test/results/clientpositive/cte_3.q.out @@ -0,0 +1,444 @@ +PREHOOK: query: explain +with q1 as ( select key from src where key = '5') +select * +from q1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as ( select key from src where key = '5') +select * +from q1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 + Stage-3 + Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 + Stage-2 + Stage-4 + Stage-5 depends on stages: Stage-4 + Stage-7 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-6 + Conditional Operator + + Stage: Stage-3 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-5 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-7 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: q1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + ListSink + +PREHOOK: query: with q1 as ( select key from src where key = '5') +select * +from q1 +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '5') +select * +from q1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: -- in subquery +explain +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +POSTHOOK: query: -- in subquery +explain +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 + Stage-3 + Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 + Stage-2 + Stage-4 + Stage-5 depends on stages: Stage-4 + Stage-7 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-6 + Conditional Operator + + Stage: Stage-3 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-5 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-7 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: q1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + ListSink + +PREHOOK: query: with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: -- chaining +explain +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +POSTHOOK: query: -- chaining +explain +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 + Stage-3 + Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 + Stage-8 depends on stages: Stage-0 + Stage-13 depends on stages: Stage-8 , consists of Stage-10, Stage-9, Stage-11 + Stage-10 + Stage-7 depends on stages: Stage-10, Stage-9, Stage-12 + Stage-9 + Stage-11 + Stage-12 depends on stages: Stage-11 + Stage-2 + Stage-4 + Stage-5 depends on stages: Stage-4 + Stage-14 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q2 + + Stage: Stage-6 + Conditional Operator + + Stage: Stage-3 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-8 + Map Reduce + Map Operator Tree: + TableScan + alias: q2 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: '5' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-13 + Conditional Operator + + Stage: Stage-10 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-7 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-9 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-11 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-12 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q2 + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q2 + + Stage: Stage-5 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-14 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: q1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + ListSink + +PREHOOK: query: with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@q2 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +PREHOOK: Output: default@q2 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@q2 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +POSTHOOK: Output: default@q2 +#### A masked pattern was here #### +5 +5 +5 diff --git ql/src/test/results/clientpositive/cte_4.q.out ql/src/test/results/clientpositive/cte_4.q.out new file mode 100644 index 0000000..d560d74 --- /dev/null +++ ql/src/test/results/clientpositive/cte_4.q.out @@ -0,0 +1,219 @@ +PREHOOK: query: -- union test +with q1 as (select * from src where key= '5'), +q2 as (select * from src s2 where key = '4') +select * from q1 union all select * from q2 +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@q2 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +PREHOOK: Output: default@q2 +#### A masked pattern was here #### +POSTHOOK: query: -- union test +with q1 as (select * from src where key= '5'), +q2 as (select * from src s2 where key = '4') +select * from q1 union all select * from q2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@q2 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +POSTHOOK: Output: default@q2 +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +4 val_4 +PREHOOK: query: -- insert test +create table s1 like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@s1 +POSTHOOK: query: -- insert test +create table s1 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@s1 +PREHOOK: query: with q1 as ( select key, value from src where key = '5') +from q1 +insert overwrite table s1 +select * +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +PREHOOK: Output: default@s1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key, value from src where key = '5') +from q1 +insert overwrite table s1 +select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +POSTHOOK: Output: default@s1 +#### A masked pattern was here #### +POSTHOOK: Lineage: s1.key SIMPLE [(q1)q1.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: s1.value SIMPLE [(q1)q1.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from s1 +PREHOOK: type: QUERY +PREHOOK: Input: default@s1 +#### A masked pattern was here #### +POSTHOOK: query: select * from s1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@s1 +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +PREHOOK: query: drop table s1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@s1 +PREHOOK: Output: default@s1 +POSTHOOK: query: drop table s1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@s1 +POSTHOOK: Output: default@s1 +PREHOOK: query: -- from style +with q1 as (select * from src where key= '5') +from q1 +select * +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: -- from style +with q1 as (select * from src where key= '5') +from q1 +select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +PREHOOK: query: -- ctas +create table s2 as +with q1 as ( select key from src where key = '4') +select * from q1 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +PREHOOK: Output: default@s2 +#### A masked pattern was here #### +POSTHOOK: query: -- ctas +create table s2 as +with q1 as ( select key from src where key = '4') +select * from q1 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +POSTHOOK: Output: default@s2 +#### A masked pattern was here #### +PREHOOK: query: select * from s2 +PREHOOK: type: QUERY +PREHOOK: Input: default@s2 +#### A masked pattern was here #### +POSTHOOK: query: select * from s2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@s2 +#### A masked pattern was here #### +4 +PREHOOK: query: drop table s2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@s2 +PREHOOK: Output: default@s2 +POSTHOOK: query: drop table s2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@s2 +POSTHOOK: Output: default@s2 +PREHOOK: query: -- view test +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v1 +POSTHOOK: query: -- view test +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v1 +PREHOOK: query: select * from v1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: drop view v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v1 +PREHOOK: Output: default@v1 +POSTHOOK: query: drop view v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v1 +POSTHOOK: Output: default@v1 +PREHOOK: query: -- view test, name collision +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v1 +POSTHOOK: query: -- view test, name collision +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v1 +PREHOOK: query: with q1 as ( select key from src where key = '4') +select * from v1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '4') +select * from v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: drop view v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v1 +PREHOOK: Output: default@v1 +POSTHOOK: query: drop view v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v1 +POSTHOOK: Output: default@v1 diff --git ql/src/test/results/clientpositive/cte_5.q.out ql/src/test/results/clientpositive/cte_5.q.out new file mode 100644 index 0000000..44a3282 --- /dev/null +++ ql/src/test/results/clientpositive/cte_5.q.out @@ -0,0 +1,156 @@ +PREHOOK: query: create database mydb +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:mydb +POSTHOOK: query: create database mydb +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:mydb +PREHOOK: query: use mydb +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:mydb +POSTHOOK: query: use mydb +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:mydb +PREHOOK: query: create table q1 (colnum int, colstring string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:mydb +PREHOOK: Output: mydb@q1 +POSTHOOK: query: create table q1 (colnum int, colstring string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:mydb +POSTHOOK: Output: mydb@q1 +PREHOOK: query: insert into q1 values (5, 'A') +PREHOOK: type: QUERY +PREHOOK: Input: mydb@values__tmp__table__1 +PREHOOK: Output: mydb@q1 +POSTHOOK: query: insert into q1 values (5, 'A') +POSTHOOK: type: QUERY +POSTHOOK: Input: mydb@values__tmp__table__1 +POSTHOOK: Output: mydb@q1 +POSTHOOK: Lineage: q1.colnum EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: q1.colstring SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: use default +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:default +POSTHOOK: query: use default +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:default +PREHOOK: query: show tables in mydb +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:mydb +POSTHOOK: query: show tables in mydb +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:mydb +q1 +values__tmp__table__1 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: colnum is not null (type: boolean) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: colnum (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 5.0 (type: double) + sort order: + + Map-reduce partition columns: 5.0 (type: double) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 UDFToDouble(_col0) (type: double) + 1 UDFToDouble('5') (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: mydb@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: mydb@q1 +#### A masked pattern was here #### +5 +5 +5 diff --git ql/src/test/results/clientpositive/cte_mat_1.q.out ql/src/test/results/clientpositive/cte_mat_1.q.out new file mode 100644 index 0000000..6429470 --- /dev/null +++ ql/src/test/results/clientpositive/cte_mat_1.q.out @@ -0,0 +1,72 @@ +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '5' (type: string) + sort order: + + Map-reduce partition columns: '5' (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '5' (type: string) + sort order: + + Map-reduce partition columns: '5' (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 '5' (type: string) + 1 '5' (type: string) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/cte_mat_2.q.out ql/src/test/results/clientpositive/cte_mat_2.q.out new file mode 100644 index 0000000..6429470 --- /dev/null +++ ql/src/test/results/clientpositive/cte_mat_2.q.out @@ -0,0 +1,72 @@ +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '5' (type: string) + sort order: + + Map-reduce partition columns: '5' (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '5' (type: string) + sort order: + + Map-reduce partition columns: '5' (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 '5' (type: string) + 1 '5' (type: string) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/cte_mat_3.q.out ql/src/test/results/clientpositive/cte_mat_3.q.out new file mode 100644 index 0000000..683228c --- /dev/null +++ ql/src/test/results/clientpositive/cte_mat_3.q.out @@ -0,0 +1,147 @@ +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 + Stage-3 + Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 + Stage-8 depends on stages: Stage-0 + Stage-2 + Stage-4 + Stage-5 depends on stages: Stage-4 + Stage-7 depends on stages: Stage-8 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-6 + Conditional Operator + + Stage: Stage-3 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-8 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-5 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-7 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/cte_mat_4.q.out ql/src/test/results/clientpositive/cte_mat_4.q.out new file mode 100644 index 0000000..048e4b6 --- /dev/null +++ ql/src/test/results/clientpositive/cte_mat_4.q.out @@ -0,0 +1,477 @@ +PREHOOK: query: create temporary table q1 (a int, b string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +POSTHOOK: query: create temporary table q1 (a int, b string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +PREHOOK: query: insert into q1 values (1, 'A') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@q1 +POSTHOOK: query: insert into q1 values (1, 'A') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@q1 +POSTHOOK: Lineage: q1.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: q1.b SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +q1 +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +values__tmp__table__1 +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 + Stage-3 + Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 + Stage-8 depends on stages: Stage-0 + Stage-2 + Stage-4 + Stage-5 depends on stages: Stage-4 + Stage-7 depends on stages: Stage-8 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-6 + Conditional Operator + + Stage: Stage-3 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-8 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-5 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-7 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 +5 +5 +5 +5 +5 +5 +5 +5 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +q1 +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +values__tmp__table__1 +PREHOOK: query: select * from q1 +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: select * from q1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +#### A masked pattern was here #### +1 A +PREHOOK: query: drop table q1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@q1 +PREHOOK: Output: default@q1 +POSTHOOK: query: drop table q1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@q1 +POSTHOOK: Output: default@q1 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +values__tmp__table__1 +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 + Stage-3 + Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 + Stage-8 depends on stages: Stage-0 + Stage-2 + Stage-4 + Stage-5 depends on stages: Stage-4 + Stage-7 depends on stages: Stage-8 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-6 + Conditional Operator + + Stage: Stage-3 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-8 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-5 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-7 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 +5 +5 +5 +5 +5 +5 +5 +5 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +values__tmp__table__1 diff --git ql/src/test/results/clientpositive/cte_mat_5.q.out ql/src/test/results/clientpositive/cte_mat_5.q.out new file mode 100644 index 0000000..b7b34ba --- /dev/null +++ ql/src/test/results/clientpositive/cte_mat_5.q.out @@ -0,0 +1,238 @@ +PREHOOK: query: create database mydb +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:mydb +POSTHOOK: query: create database mydb +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:mydb +PREHOOK: query: use mydb +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:mydb +POSTHOOK: query: use mydb +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:mydb +PREHOOK: query: create table q1 (colnum int, colstring string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:mydb +PREHOOK: Output: mydb@q1 +POSTHOOK: query: create table q1 (colnum int, colstring string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:mydb +POSTHOOK: Output: mydb@q1 +PREHOOK: query: insert into q1 values (5, 'A') +PREHOOK: type: QUERY +PREHOOK: Input: mydb@values__tmp__table__1 +PREHOOK: Output: mydb@q1 +POSTHOOK: query: insert into q1 values (5, 'A') +POSTHOOK: type: QUERY +POSTHOOK: Input: mydb@values__tmp__table__1 +POSTHOOK: Output: mydb@q1 +POSTHOOK: Lineage: q1.colnum EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: q1.colstring SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: use default +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:default +POSTHOOK: query: use default +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:default +PREHOOK: query: show tables in mydb +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:mydb +POSTHOOK: query: show tables in mydb +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:mydb +q1 +values__tmp__table__1 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 + Stage-3 + Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 + Stage-8 depends on stages: Stage-0 + Stage-2 + Stage-4 + Stage-5 depends on stages: Stage-4 + Stage-7 depends on stages: Stage-8 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-6 + Conditional Operator + + Stage: Stage-3 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-8 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: colnum is not null (type: boolean) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: colnum (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + TableScan + alias: b + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 UDFToDouble(_col0) (type: double) + 1 UDFToDouble(_col0) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + + Stage: Stage-5 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-7 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Input: mydb@q1 +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Input: mydb@q1 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 +5 +5 diff --git ql/src/test/results/clientpositive/llap/cte_1.q.out ql/src/test/results/clientpositive/llap/cte_1.q.out new file mode 100644 index 0000000..41ffdc7 --- /dev/null +++ ql/src/test/results/clientpositive/llap/cte_1.q.out @@ -0,0 +1,126 @@ +PREHOOK: query: explain +with q1 as ( select key from src where key = '5') +select * +from q1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as ( select key from src where key = '5') +select * +from q1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src + Filter Operator + predicate: (key = '5') (type: boolean) + Select Operator + expressions: '5' (type: string) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: with q1 as ( select key from src where key = '5') +select * +from q1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '5') +select * +from q1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: -- in subquery +explain +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +POSTHOOK: query: -- in subquery +explain +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src + Filter Operator + predicate: (key = '5') (type: boolean) + Select Operator + expressions: '5' (type: string) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: -- chaining +explain +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +POSTHOOK: query: -- chaining +explain +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src + Filter Operator + predicate: (key = '5') (type: boolean) + Select Operator + expressions: '5' (type: string) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +5 +5 +5 diff --git ql/src/test/results/clientpositive/llap/cte_2.q.out ql/src/test/results/clientpositive/llap/cte_2.q.out new file mode 100644 index 0000000..23f8ec6 --- /dev/null +++ ql/src/test/results/clientpositive/llap/cte_2.q.out @@ -0,0 +1,189 @@ +PREHOOK: query: -- union test +with q1 as (select * from src where key= '5'), +q2 as (select * from src s2 where key = '4') +select * from q1 union all select * from q2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- union test +with q1 as (select * from src where key= '5'), +q2 as (select * from src s2 where key = '4') +select * from q1 union all select * from q2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +4 val_4 +PREHOOK: query: -- insert test +create table s1 like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@s1 +POSTHOOK: query: -- insert test +create table s1 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@s1 +PREHOOK: query: with q1 as ( select key, value from src where key = '5') +from q1 +insert overwrite table s1 +select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@s1 +POSTHOOK: query: with q1 as ( select key, value from src where key = '5') +from q1 +insert overwrite table s1 +select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@s1 +POSTHOOK: Lineage: s1.key SIMPLE [] +POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from s1 +PREHOOK: type: QUERY +PREHOOK: Input: default@s1 +#### A masked pattern was here #### +POSTHOOK: query: select * from s1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@s1 +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +PREHOOK: query: drop table s1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@s1 +PREHOOK: Output: default@s1 +POSTHOOK: query: drop table s1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@s1 +POSTHOOK: Output: default@s1 +PREHOOK: query: -- from style +with q1 as (select * from src where key= '5') +from q1 +select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- from style +with q1 as (select * from src where key= '5') +from q1 +select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +PREHOOK: query: -- ctas +create table s2 as +with q1 as ( select key from src where key = '4') +select * from q1 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@s2 +POSTHOOK: query: -- ctas +create table s2 as +with q1 as ( select key from src where key = '4') +select * from q1 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@s2 +PREHOOK: query: select * from s2 +PREHOOK: type: QUERY +PREHOOK: Input: default@s2 +#### A masked pattern was here #### +POSTHOOK: query: select * from s2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@s2 +#### A masked pattern was here #### +4 +PREHOOK: query: drop table s2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@s2 +PREHOOK: Output: default@s2 +POSTHOOK: query: drop table s2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@s2 +POSTHOOK: Output: default@s2 +PREHOOK: query: -- view test +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v1 +POSTHOOK: query: -- view test +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v1 +PREHOOK: query: select * from v1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: drop view v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v1 +PREHOOK: Output: default@v1 +POSTHOOK: query: drop view v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v1 +POSTHOOK: Output: default@v1 +PREHOOK: query: -- view test, name collision +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v1 +POSTHOOK: query: -- view test, name collision +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v1 +PREHOOK: query: with q1 as ( select key from src where key = '4') +select * from v1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '4') +select * from v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: drop view v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v1 +PREHOOK: Output: default@v1 +POSTHOOK: query: drop view v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v1 +POSTHOOK: Output: default@v1 diff --git ql/src/test/results/clientpositive/llap/cte_3.q.out ql/src/test/results/clientpositive/llap/cte_3.q.out new file mode 100644 index 0000000..37796c0 --- /dev/null +++ ql/src/test/results/clientpositive/llap/cte_3.q.out @@ -0,0 +1,294 @@ +PREHOOK: query: explain +with q1 as ( select key from src where key = '5') +select * +from q1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as ( select key from src where key = '5') +select * +from q1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-1 + Stage-3 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + Execution mode: llap + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-3 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: q1 + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: with q1 as ( select key from src where key = '5') +select * +from q1 +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '5') +select * +from q1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: -- in subquery +explain +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +POSTHOOK: query: -- in subquery +explain +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-1 + Stage-3 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + Execution mode: llap + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-3 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: q1 + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: -- chaining +explain +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +POSTHOOK: query: -- chaining +explain +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2, Stage-0 + Stage-5 depends on stages: Stage-4 + Stage-3 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-1 + Stage-6 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q2 + Execution mode: llap + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Tez +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: q2 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: '5' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + Execution mode: llap + + Stage: Stage-5 + Dependency Collection + + Stage: Stage-3 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-6 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: q1 + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@q2 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +PREHOOK: Output: default@q2 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@q2 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +POSTHOOK: Output: default@q2 +#### A masked pattern was here #### +5 +5 +5 diff --git ql/src/test/results/clientpositive/llap/cte_4.q.out ql/src/test/results/clientpositive/llap/cte_4.q.out new file mode 100644 index 0000000..d560d74 --- /dev/null +++ ql/src/test/results/clientpositive/llap/cte_4.q.out @@ -0,0 +1,219 @@ +PREHOOK: query: -- union test +with q1 as (select * from src where key= '5'), +q2 as (select * from src s2 where key = '4') +select * from q1 union all select * from q2 +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@q2 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +PREHOOK: Output: default@q2 +#### A masked pattern was here #### +POSTHOOK: query: -- union test +with q1 as (select * from src where key= '5'), +q2 as (select * from src s2 where key = '4') +select * from q1 union all select * from q2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@q2 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +POSTHOOK: Output: default@q2 +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +4 val_4 +PREHOOK: query: -- insert test +create table s1 like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@s1 +POSTHOOK: query: -- insert test +create table s1 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@s1 +PREHOOK: query: with q1 as ( select key, value from src where key = '5') +from q1 +insert overwrite table s1 +select * +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +PREHOOK: Output: default@s1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key, value from src where key = '5') +from q1 +insert overwrite table s1 +select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +POSTHOOK: Output: default@s1 +#### A masked pattern was here #### +POSTHOOK: Lineage: s1.key SIMPLE [(q1)q1.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: s1.value SIMPLE [(q1)q1.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from s1 +PREHOOK: type: QUERY +PREHOOK: Input: default@s1 +#### A masked pattern was here #### +POSTHOOK: query: select * from s1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@s1 +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +PREHOOK: query: drop table s1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@s1 +PREHOOK: Output: default@s1 +POSTHOOK: query: drop table s1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@s1 +POSTHOOK: Output: default@s1 +PREHOOK: query: -- from style +with q1 as (select * from src where key= '5') +from q1 +select * +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: -- from style +with q1 as (select * from src where key= '5') +from q1 +select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +PREHOOK: query: -- ctas +create table s2 as +with q1 as ( select key from src where key = '4') +select * from q1 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +PREHOOK: Output: default@s2 +#### A masked pattern was here #### +POSTHOOK: query: -- ctas +create table s2 as +with q1 as ( select key from src where key = '4') +select * from q1 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +POSTHOOK: Output: default@s2 +#### A masked pattern was here #### +PREHOOK: query: select * from s2 +PREHOOK: type: QUERY +PREHOOK: Input: default@s2 +#### A masked pattern was here #### +POSTHOOK: query: select * from s2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@s2 +#### A masked pattern was here #### +4 +PREHOOK: query: drop table s2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@s2 +PREHOOK: Output: default@s2 +POSTHOOK: query: drop table s2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@s2 +POSTHOOK: Output: default@s2 +PREHOOK: query: -- view test +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v1 +POSTHOOK: query: -- view test +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v1 +PREHOOK: query: select * from v1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: drop view v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v1 +PREHOOK: Output: default@v1 +POSTHOOK: query: drop view v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v1 +POSTHOOK: Output: default@v1 +PREHOOK: query: -- view test, name collision +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v1 +POSTHOOK: query: -- view test, name collision +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v1 +PREHOOK: query: with q1 as ( select key from src where key = '4') +select * from v1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '4') +select * from v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: drop view v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v1 +PREHOOK: Output: default@v1 +POSTHOOK: query: drop view v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v1 +POSTHOOK: Output: default@v1 diff --git ql/src/test/results/clientpositive/llap/cte_5.q.out ql/src/test/results/clientpositive/llap/cte_5.q.out new file mode 100644 index 0000000..3092398 --- /dev/null +++ ql/src/test/results/clientpositive/llap/cte_5.q.out @@ -0,0 +1,168 @@ +PREHOOK: query: create database mydb +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:mydb +POSTHOOK: query: create database mydb +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:mydb +PREHOOK: query: use mydb +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:mydb +POSTHOOK: query: use mydb +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:mydb +PREHOOK: query: create table q1 (colnum int, colstring string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:mydb +PREHOOK: Output: mydb@q1 +POSTHOOK: query: create table q1 (colnum int, colstring string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:mydb +POSTHOOK: Output: mydb@q1 +PREHOOK: query: insert into q1 values (5, 'A') +PREHOOK: type: QUERY +PREHOOK: Input: mydb@values__tmp__table__1 +PREHOOK: Output: mydb@q1 +POSTHOOK: query: insert into q1 values (5, 'A') +POSTHOOK: type: QUERY +POSTHOOK: Input: mydb@values__tmp__table__1 +POSTHOOK: Output: mydb@q1 +POSTHOOK: Lineage: q1.colnum EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: q1.colstring SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: use default +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:default +POSTHOOK: query: use default +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:default +PREHOOK: query: show tables in mydb +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:mydb +POSTHOOK: query: show tables in mydb +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:mydb +q1 +values__tmp__table__1 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: colnum is not null (type: boolean) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: colnum (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + Map 3 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 5.0 (type: double) + sort order: + + Map-reduce partition columns: 5.0 (type: double) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 UDFToDouble(_col0) (type: double) + 1 5.0 (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: mydb@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: mydb@q1 +#### A masked pattern was here #### +5 +5 +5 diff --git ql/src/test/results/clientpositive/llap/cte_mat_1.q.out ql/src/test/results/clientpositive/llap/cte_mat_1.q.out new file mode 100644 index 0000000..06135d7 --- /dev/null +++ ql/src/test/results/clientpositive/llap/cte_mat_1.q.out @@ -0,0 +1,83 @@ +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '5' (type: string) + sort order: + + Map-reduce partition columns: '5' (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + Map 3 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '5' (type: string) + sort order: + + Map-reduce partition columns: '5' (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 '5' (type: string) + 1 '5' (type: string) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/llap/cte_mat_2.q.out ql/src/test/results/clientpositive/llap/cte_mat_2.q.out new file mode 100644 index 0000000..06135d7 --- /dev/null +++ ql/src/test/results/clientpositive/llap/cte_mat_2.q.out @@ -0,0 +1,83 @@ +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '5' (type: string) + sort order: + + Map-reduce partition columns: '5' (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + Map 3 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '5' (type: string) + sort order: + + Map-reduce partition columns: '5' (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 '5' (type: string) + 1 '5' (type: string) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/llap/cte_mat_3.q.out ql/src/test/results/clientpositive/llap/cte_mat_3.q.out new file mode 100644 index 0000000..cade4c9 --- /dev/null +++ ql/src/test/results/clientpositive/llap/cte_mat_3.q.out @@ -0,0 +1,122 @@ +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2, Stage-0 + Stage-0 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-4 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + Execution mode: llap + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Tez +#### A masked pattern was here #### + Edges: + Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: llap + Map 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: llap + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-3 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git ql/src/test/results/clientpositive/llap/cte_mat_4.q.out ql/src/test/results/clientpositive/llap/cte_mat_4.q.out new file mode 100644 index 0000000..1a53b43 --- /dev/null +++ ql/src/test/results/clientpositive/llap/cte_mat_4.q.out @@ -0,0 +1,429 @@ +PREHOOK: query: create temporary table q1 (a int, b string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +POSTHOOK: query: create temporary table q1 (a int, b string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +PREHOOK: query: insert into q1 values (1, 'A') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@q1 +POSTHOOK: query: insert into q1 values (1, 'A') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@q1 +POSTHOOK: Lineage: q1.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: q1.b SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +q1 +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +values__tmp__table__1 +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2, Stage-0 + Stage-0 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-4 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + Execution mode: llap + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Tez +#### A masked pattern was here #### + Edges: + Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: llap + Map 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: llap + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-3 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 +5 +5 +5 +5 +5 +5 +5 +5 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +q1 +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +values__tmp__table__1 +PREHOOK: query: select * from q1 +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: select * from q1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +#### A masked pattern was here #### +1 A +PREHOOK: query: drop table q1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@q1 +PREHOOK: Output: default@q1 +POSTHOOK: query: drop table q1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@q1 +POSTHOOK: Output: default@q1 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +values__tmp__table__1 +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2, Stage-0 + Stage-0 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-4 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + Execution mode: llap + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Tez +#### A masked pattern was here #### + Edges: + Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: llap + Map 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: llap + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-3 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 +5 +5 +5 +5 +5 +5 +5 +5 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +values__tmp__table__1 diff --git ql/src/test/results/clientpositive/llap/cte_mat_5.q.out ql/src/test/results/clientpositive/llap/cte_mat_5.q.out new file mode 100644 index 0000000..9fa3e7d --- /dev/null +++ ql/src/test/results/clientpositive/llap/cte_mat_5.q.out @@ -0,0 +1,214 @@ +PREHOOK: query: create database mydb +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:mydb +POSTHOOK: query: create database mydb +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:mydb +PREHOOK: query: use mydb +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:mydb +POSTHOOK: query: use mydb +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:mydb +PREHOOK: query: create table q1 (colnum int, colstring string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:mydb +PREHOOK: Output: mydb@q1 +POSTHOOK: query: create table q1 (colnum int, colstring string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:mydb +POSTHOOK: Output: mydb@q1 +PREHOOK: query: insert into q1 values (5, 'A') +PREHOOK: type: QUERY +PREHOOK: Input: mydb@values__tmp__table__1 +PREHOOK: Output: mydb@q1 +POSTHOOK: query: insert into q1 values (5, 'A') +POSTHOOK: type: QUERY +POSTHOOK: Input: mydb@values__tmp__table__1 +POSTHOOK: Output: mydb@q1 +POSTHOOK: Lineage: q1.colnum EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: q1.colstring SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: use default +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:default +POSTHOOK: query: use default +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:default +PREHOOK: query: show tables in mydb +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:mydb +POSTHOOK: query: show tables in mydb +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:mydb +q1 +values__tmp__table__1 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2, Stage-0 + Stage-0 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-4 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key = '5') (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '5' (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.q1 + Execution mode: llap + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Tez +#### A masked pattern was here #### + Edges: + Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 2 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: colnum is not null (type: boolean) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: colnum (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: llap + Map 4 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: llap + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 UDFToDouble(_col0) (type: double) + 1 UDFToDouble(_col0) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-3 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Input: mydb@q1 +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Input: mydb@q1 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 +5 +5 diff --git ql/src/test/results/clientpositive/llap/tez_union.q.out ql/src/test/results/clientpositive/llap/tez_union.q.out index f5cd30f..8e69868 100644 --- ql/src/test/results/clientpositive/llap/tez_union.q.out +++ ql/src/test/results/clientpositive/llap/tez_union.q.out @@ -1436,7 +1436,6 @@ POSTHOOK: Input: default@table1 POSTHOOK: Input: default@table2 POSTHOOK: Output: database:default POSTHOOK: Output: default@TABLE3 -POSTHOOK: Output: default@table3 PREHOOK: query: explain formatted select count(*) from TABLE3 PREHOOK: type: QUERY POSTHOOK: query: explain formatted select count(*) from TABLE3 diff --git ql/src/test/results/clientpositive/tez/cte_1.q.out ql/src/test/results/clientpositive/tez/cte_1.q.out new file mode 100644 index 0000000..1b24fb0 --- /dev/null +++ ql/src/test/results/clientpositive/tez/cte_1.q.out @@ -0,0 +1,111 @@ +PREHOOK: query: explain +with q1 as ( select key from src where key = '5') +select * +from q1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as ( select key from src where key = '5') +select * +from q1 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-0 + Fetch Operator + limit:-1 + Select Operator [SEL_2] + Output:["_col0"] + Filter Operator [FIL_4] + predicate:(key = '5') + TableScan [TS_0] + Output:["key"] + +PREHOOK: query: with q1 as ( select key from src where key = '5') +select * +from q1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '5') +select * +from q1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: -- in subquery +explain +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +POSTHOOK: query: -- in subquery +explain +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-0 + Fetch Operator + limit:-1 + Select Operator [SEL_2] + Output:["_col0"] + Filter Operator [FIL_4] + predicate:(key = '5') + TableScan [TS_0] + Output:["key"] + +PREHOOK: query: with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: -- chaining +explain +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +POSTHOOK: query: -- chaining +explain +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-0 + Fetch Operator + limit:-1 + Select Operator [SEL_2] + Output:["_col0"] + Filter Operator [FIL_4] + predicate:(key = '5') + TableScan [TS_0] + Output:["key"] + +PREHOOK: query: with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +5 +5 +5 diff --git ql/src/test/results/clientpositive/tez/cte_2.q.out ql/src/test/results/clientpositive/tez/cte_2.q.out new file mode 100644 index 0000000..23f8ec6 --- /dev/null +++ ql/src/test/results/clientpositive/tez/cte_2.q.out @@ -0,0 +1,189 @@ +PREHOOK: query: -- union test +with q1 as (select * from src where key= '5'), +q2 as (select * from src s2 where key = '4') +select * from q1 union all select * from q2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- union test +with q1 as (select * from src where key= '5'), +q2 as (select * from src s2 where key = '4') +select * from q1 union all select * from q2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +4 val_4 +PREHOOK: query: -- insert test +create table s1 like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@s1 +POSTHOOK: query: -- insert test +create table s1 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@s1 +PREHOOK: query: with q1 as ( select key, value from src where key = '5') +from q1 +insert overwrite table s1 +select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@s1 +POSTHOOK: query: with q1 as ( select key, value from src where key = '5') +from q1 +insert overwrite table s1 +select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@s1 +POSTHOOK: Lineage: s1.key SIMPLE [] +POSTHOOK: Lineage: s1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from s1 +PREHOOK: type: QUERY +PREHOOK: Input: default@s1 +#### A masked pattern was here #### +POSTHOOK: query: select * from s1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@s1 +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +PREHOOK: query: drop table s1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@s1 +PREHOOK: Output: default@s1 +POSTHOOK: query: drop table s1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@s1 +POSTHOOK: Output: default@s1 +PREHOOK: query: -- from style +with q1 as (select * from src where key= '5') +from q1 +select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- from style +with q1 as (select * from src where key= '5') +from q1 +select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +PREHOOK: query: -- ctas +create table s2 as +with q1 as ( select key from src where key = '4') +select * from q1 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@s2 +POSTHOOK: query: -- ctas +create table s2 as +with q1 as ( select key from src where key = '4') +select * from q1 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@s2 +PREHOOK: query: select * from s2 +PREHOOK: type: QUERY +PREHOOK: Input: default@s2 +#### A masked pattern was here #### +POSTHOOK: query: select * from s2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@s2 +#### A masked pattern was here #### +4 +PREHOOK: query: drop table s2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@s2 +PREHOOK: Output: default@s2 +POSTHOOK: query: drop table s2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@s2 +POSTHOOK: Output: default@s2 +PREHOOK: query: -- view test +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v1 +POSTHOOK: query: -- view test +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v1 +PREHOOK: query: select * from v1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: drop view v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v1 +PREHOOK: Output: default@v1 +POSTHOOK: query: drop view v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v1 +POSTHOOK: Output: default@v1 +PREHOOK: query: -- view test, name collision +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v1 +POSTHOOK: query: -- view test, name collision +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v1 +PREHOOK: query: with q1 as ( select key from src where key = '4') +select * from v1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '4') +select * from v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: drop view v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v1 +PREHOOK: Output: default@v1 +POSTHOOK: query: drop view v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v1 +POSTHOOK: Output: default@v1 diff --git ql/src/test/results/clientpositive/tez/cte_3.q.out ql/src/test/results/clientpositive/tez/cte_3.q.out new file mode 100644 index 0000000..fedbb7d --- /dev/null +++ ql/src/test/results/clientpositive/tez/cte_3.q.out @@ -0,0 +1,187 @@ +PREHOOK: query: explain +with q1 as ( select key from src where key = '5') +select * +from q1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as ( select key from src where key = '5') +select * +from q1 +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-2 + Dependency Collection{} + Stage-1 + Map 1 + File Output Operator [FS_3] + table:{"name:":"default.q1"} + Select Operator [SEL_2] (rows=250 width=10) + Output:["_col0"] + Filter Operator [FIL_4] (rows=250 width=10) + predicate:(key = '5') + TableScan [TS_0] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] +Stage-0 + Move Operator + Please refer to the previous Stage-1 +Stage-3 + Fetch Operator + limit:-1 + Select Operator [SEL_6] + Output:["_col0"] + TableScan [TS_5] + Output:["key"] + +PREHOOK: query: with q1 as ( select key from src where key = '5') +select * +from q1 +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '5') +select * +from q1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: -- in subquery +explain +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +POSTHOOK: query: -- in subquery +explain +with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-2 + Dependency Collection{} + Stage-1 + Map 1 + File Output Operator [FS_3] + table:{"name:":"default.q1"} + Select Operator [SEL_2] (rows=250 width=10) + Output:["_col0"] + Filter Operator [FIL_4] (rows=250 width=10) + predicate:(key = '5') + TableScan [TS_0] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] +Stage-0 + Move Operator + Please refer to the previous Stage-1 +Stage-3 + Fetch Operator + limit:-1 + Select Operator [SEL_6] + Output:["_col0"] + TableScan [TS_5] + Output:["key"] + +PREHOOK: query: with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: -- chaining +explain +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +POSTHOOK: query: -- chaining +explain +with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Stage-5 + Dependency Collection{} + Stage-4 + Map 2 + File Output Operator [FS_8] + table:{"name:":"default.q1"} + Select Operator [SEL_7] (rows=1 width=0) + Output:["_col0"] + Filter Operator [FIL_9] (rows=1 width=0) + predicate:(key = '5') + TableScan [TS_5] (rows=1 width=0) + default@q2,q2,Tbl:PARTIAL,Col:NONE,Output:["key"] + Stage-2 + Dependency Collection{} + Stage-1 + Map 1 + File Output Operator [FS_3] + table:{"name:":"default.q2"} + Select Operator [SEL_2] (rows=250 width=10) + Output:["_col0"] + Filter Operator [FIL_4] (rows=250 width=10) + predicate:(key = '5') + TableScan [TS_0] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] + Stage-0 + Move Operator + Please refer to the previous Stage-1 +Stage-3 + Move Operator + Please refer to the previous Stage-4 +Stage-6 + Fetch Operator + limit:-1 + Select Operator [SEL_11] + Output:["_col0"] + TableScan [TS_10] + Output:["key"] + +PREHOOK: query: with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@q2 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +PREHOOK: Output: default@q2 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from q2 where key = '5'), +q2 as ( select key from src where key = '5') +select * from (select key from q1) a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@q2 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +POSTHOOK: Output: default@q2 +#### A masked pattern was here #### +5 +5 +5 diff --git ql/src/test/results/clientpositive/tez/cte_4.q.out ql/src/test/results/clientpositive/tez/cte_4.q.out new file mode 100644 index 0000000..d560d74 --- /dev/null +++ ql/src/test/results/clientpositive/tez/cte_4.q.out @@ -0,0 +1,219 @@ +PREHOOK: query: -- union test +with q1 as (select * from src where key= '5'), +q2 as (select * from src s2 where key = '4') +select * from q1 union all select * from q2 +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@q2 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +PREHOOK: Output: default@q2 +#### A masked pattern was here #### +POSTHOOK: query: -- union test +with q1 as (select * from src where key= '5'), +q2 as (select * from src s2 where key = '4') +select * from q1 union all select * from q2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@q2 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +POSTHOOK: Output: default@q2 +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +4 val_4 +PREHOOK: query: -- insert test +create table s1 like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@s1 +POSTHOOK: query: -- insert test +create table s1 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@s1 +PREHOOK: query: with q1 as ( select key, value from src where key = '5') +from q1 +insert overwrite table s1 +select * +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +PREHOOK: Output: default@s1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key, value from src where key = '5') +from q1 +insert overwrite table s1 +select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +POSTHOOK: Output: default@s1 +#### A masked pattern was here #### +POSTHOOK: Lineage: s1.key SIMPLE [(q1)q1.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: s1.value SIMPLE [(q1)q1.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from s1 +PREHOOK: type: QUERY +PREHOOK: Input: default@s1 +#### A masked pattern was here #### +POSTHOOK: query: select * from s1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@s1 +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +PREHOOK: query: drop table s1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@s1 +PREHOOK: Output: default@s1 +POSTHOOK: query: drop table s1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@s1 +POSTHOOK: Output: default@s1 +PREHOOK: query: -- from style +with q1 as (select * from src where key= '5') +from q1 +select * +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: -- from style +with q1 as (select * from src where key= '5') +from q1 +select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 val_5 +5 val_5 +5 val_5 +PREHOOK: query: -- ctas +create table s2 as +with q1 as ( select key from src where key = '4') +select * from q1 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +PREHOOK: Output: default@s2 +#### A masked pattern was here #### +POSTHOOK: query: -- ctas +create table s2 as +with q1 as ( select key from src where key = '4') +select * from q1 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +POSTHOOK: Output: default@s2 +#### A masked pattern was here #### +PREHOOK: query: select * from s2 +PREHOOK: type: QUERY +PREHOOK: Input: default@s2 +#### A masked pattern was here #### +POSTHOOK: query: select * from s2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@s2 +#### A masked pattern was here #### +4 +PREHOOK: query: drop table s2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@s2 +PREHOOK: Output: default@s2 +POSTHOOK: query: drop table s2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@s2 +POSTHOOK: Output: default@s2 +PREHOOK: query: -- view test +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v1 +POSTHOOK: query: -- view test +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v1 +PREHOOK: query: select * from v1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: drop view v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v1 +PREHOOK: Output: default@v1 +POSTHOOK: query: drop view v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v1 +POSTHOOK: Output: default@v1 +PREHOOK: query: -- view test, name collision +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v1 +POSTHOOK: query: -- view test, name collision +create view v1 as +with q1 as ( select key from src where key = '5') +select * from q1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v1 +PREHOOK: query: with q1 as ( select key from src where key = '4') +select * from v1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as ( select key from src where key = '4') +select * from v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v1 +#### A masked pattern was here #### +5 +5 +5 +PREHOOK: query: drop view v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v1 +PREHOOK: Output: default@v1 +POSTHOOK: query: drop view v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v1 +POSTHOOK: Output: default@v1 diff --git ql/src/test/results/clientpositive/tez/cte_5.q.out ql/src/test/results/clientpositive/tez/cte_5.q.out new file mode 100644 index 0000000..579b4f3 --- /dev/null +++ ql/src/test/results/clientpositive/tez/cte_5.q.out @@ -0,0 +1,127 @@ +PREHOOK: query: create database mydb +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:mydb +POSTHOOK: query: create database mydb +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:mydb +PREHOOK: query: use mydb +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:mydb +POSTHOOK: query: use mydb +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:mydb +PREHOOK: query: create table q1 (colnum int, colstring string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:mydb +PREHOOK: Output: mydb@q1 +POSTHOOK: query: create table q1 (colnum int, colstring string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:mydb +POSTHOOK: Output: mydb@q1 +PREHOOK: query: insert into q1 values (5, 'A') +PREHOOK: type: QUERY +PREHOOK: Input: mydb@values__tmp__table__1 +PREHOOK: Output: mydb@q1 +POSTHOOK: query: insert into q1 values (5, 'A') +POSTHOOK: type: QUERY +POSTHOOK: Input: mydb@values__tmp__table__1 +POSTHOOK: Output: mydb@q1 +POSTHOOK: Lineage: q1.colnum EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: q1.colstring SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: use default +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:default +POSTHOOK: query: use default +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:default +PREHOOK: query: show tables in mydb +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:mydb +POSTHOOK: query: show tables in mydb +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:mydb +q1 +values__tmp__table__1 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_10] + Merge Join Operator [MERGEJOIN_15] (rows=275 width=10) + Conds:RS_6.UDFToDouble(_col0)=RS_7.5.0(Inner),Output:["_col0"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_6] + PartitionCols:UDFToDouble(_col0) + Select Operator [SEL_2] (rows=1 width=3) + Output:["_col0"] + Filter Operator [FIL_13] (rows=1 width=3) + predicate:colnum is not null + TableScan [TS_0] (rows=1 width=3) + mydb@q1,a,Tbl:COMPLETE,Col:NONE,Output:["colnum"] + <-Map 3 [SIMPLE_EDGE] + SHUFFLE [RS_7] + PartitionCols:5.0 + Select Operator [SEL_5] (rows=250 width=10) + Filter Operator [FIL_14] (rows=250 width=10) + predicate:(key = '5') + TableScan [TS_3] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] + +PREHOOK: query: with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: mydb@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: mydb@q1 +#### A masked pattern was here #### +5 +5 +5 diff --git ql/src/test/results/clientpositive/tez/cte_mat_1.q.out ql/src/test/results/clientpositive/tez/cte_mat_1.q.out new file mode 100644 index 0000000..bbe4296 --- /dev/null +++ ql/src/test/results/clientpositive/tez/cte_mat_1.q.out @@ -0,0 +1,44 @@ +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_10] + Select Operator [SEL_9] (rows=275 width=10) + Output:["_col0"] + Merge Join Operator [MERGEJOIN_15] (rows=275 width=10) + Conds:RS_6.'5'=RS_7.'5'(Inner) + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_6] + PartitionCols:'5' + Select Operator [SEL_2] (rows=250 width=10) + Filter Operator [FIL_13] (rows=250 width=10) + predicate:(key = '5') + TableScan [TS_0] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] + <-Map 3 [SIMPLE_EDGE] + SHUFFLE [RS_7] + PartitionCols:'5' + Select Operator [SEL_5] (rows=250 width=10) + Filter Operator [FIL_14] (rows=250 width=10) + predicate:(key = '5') + TableScan [TS_3] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] + diff --git ql/src/test/results/clientpositive/tez/cte_mat_2.q.out ql/src/test/results/clientpositive/tez/cte_mat_2.q.out new file mode 100644 index 0000000..bbe4296 --- /dev/null +++ ql/src/test/results/clientpositive/tez/cte_mat_2.q.out @@ -0,0 +1,44 @@ +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_10] + Select Operator [SEL_9] (rows=275 width=10) + Output:["_col0"] + Merge Join Operator [MERGEJOIN_15] (rows=275 width=10) + Conds:RS_6.'5'=RS_7.'5'(Inner) + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_6] + PartitionCols:'5' + Select Operator [SEL_2] (rows=250 width=10) + Filter Operator [FIL_13] (rows=250 width=10) + predicate:(key = '5') + TableScan [TS_0] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] + <-Map 3 [SIMPLE_EDGE] + SHUFFLE [RS_7] + PartitionCols:'5' + Select Operator [SEL_5] (rows=250 width=10) + Filter Operator [FIL_14] (rows=250 width=10) + predicate:(key = '5') + TableScan [TS_3] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"] + diff --git ql/src/test/results/clientpositive/tez/cte_mat_3.q.out ql/src/test/results/clientpositive/tez/cte_mat_3.q.out new file mode 100644 index 0000000..b700d44 --- /dev/null +++ ql/src/test/results/clientpositive/tez/cte_mat_3.q.out @@ -0,0 +1,59 @@ +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + +Stage-3 + Fetch Operator + limit:-1 + Stage-4 + Reducer 3 + File Output Operator [FS_15] + Merge Join Operator [MERGEJOIN_20] (rows=1 width=0) + Conds:RS_11._col0=RS_12._col0(Inner),Output:["_col0"] + <-Map 2 [SIMPLE_EDGE] + SHUFFLE [RS_11] + PartitionCols:_col0 + Select Operator [SEL_7] (rows=1 width=0) + Output:["_col0"] + Filter Operator [FIL_18] (rows=1 width=0) + predicate:key is not null + TableScan [TS_5] (rows=1 width=0) + default@q1,a,Tbl:PARTIAL,Col:NONE,Output:["key"] + <-Map 4 [SIMPLE_EDGE] + SHUFFLE [RS_12] + PartitionCols:_col0 + Select Operator [SEL_10] (rows=1 width=0) + Output:["_col0"] + Filter Operator [FIL_19] (rows=1 width=0) + predicate:key is not null + TableScan [TS_8] (rows=1 width=0) + default@q1,a,Tbl:PARTIAL,Col:NONE,Output:["key"] + Stage-2 + Dependency Collection{} + Stage-1 + Map 1 + File Output Operator [FS_3] + table:{"name:":"default.q1"} + Select Operator [SEL_2] (rows=250 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_4] (rows=250 width=10) + predicate:(key = '5') + TableScan [TS_0] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Stage-0 + Move Operator + Please refer to the previous Stage-1 + diff --git ql/src/test/results/clientpositive/tez/cte_mat_4.q.out ql/src/test/results/clientpositive/tez/cte_mat_4.q.out new file mode 100644 index 0000000..bf13958 --- /dev/null +++ ql/src/test/results/clientpositive/tez/cte_mat_4.q.out @@ -0,0 +1,301 @@ +PREHOOK: query: create temporary table q1 (a int, b string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +POSTHOOK: query: create temporary table q1 (a int, b string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +PREHOOK: query: insert into q1 values (1, 'A') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@q1 +POSTHOOK: query: insert into q1 values (1, 'A') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@q1 +POSTHOOK: Lineage: q1.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: q1.b SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +q1 +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +values__tmp__table__1 +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + +Stage-3 + Fetch Operator + limit:-1 + Stage-4 + Reducer 3 + File Output Operator [FS_15] + Merge Join Operator [MERGEJOIN_20] (rows=1 width=0) + Conds:RS_11._col0=RS_12._col0(Inner),Output:["_col0"] + <-Map 2 [SIMPLE_EDGE] + SHUFFLE [RS_11] + PartitionCols:_col0 + Select Operator [SEL_7] (rows=1 width=0) + Output:["_col0"] + Filter Operator [FIL_18] (rows=1 width=0) + predicate:key is not null + TableScan [TS_5] (rows=1 width=0) + default@q1,a,Tbl:PARTIAL,Col:NONE,Output:["key"] + <-Map 4 [SIMPLE_EDGE] + SHUFFLE [RS_12] + PartitionCols:_col0 + Select Operator [SEL_10] (rows=1 width=0) + Output:["_col0"] + Filter Operator [FIL_19] (rows=1 width=0) + predicate:key is not null + TableScan [TS_8] (rows=1 width=0) + default@q1,a,Tbl:PARTIAL,Col:NONE,Output:["key"] + Stage-2 + Dependency Collection{} + Stage-1 + Map 1 + File Output Operator [FS_3] + table:{"name:":"default.q1"} + Select Operator [SEL_2] (rows=250 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_4] (rows=250 width=10) + predicate:(key = '5') + TableScan [TS_0] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Stage-0 + Move Operator + Please refer to the previous Stage-1 + +PREHOOK: query: with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 +5 +5 +5 +5 +5 +5 +5 +5 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +q1 +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +values__tmp__table__1 +PREHOOK: query: select * from q1 +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: select * from q1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +#### A masked pattern was here #### +1 A +PREHOOK: query: drop table q1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@q1 +PREHOOK: Output: default@q1 +POSTHOOK: query: drop table q1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@q1 +POSTHOOK: Output: default@q1 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +values__tmp__table__1 +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + +Stage-3 + Fetch Operator + limit:-1 + Stage-4 + Reducer 3 + File Output Operator [FS_15] + Merge Join Operator [MERGEJOIN_20] (rows=1 width=0) + Conds:RS_11._col0=RS_12._col0(Inner),Output:["_col0"] + <-Map 2 [SIMPLE_EDGE] + SHUFFLE [RS_11] + PartitionCols:_col0 + Select Operator [SEL_7] (rows=1 width=0) + Output:["_col0"] + Filter Operator [FIL_18] (rows=1 width=0) + predicate:key is not null + TableScan [TS_5] (rows=1 width=0) + default@q1,a,Tbl:PARTIAL,Col:NONE,Output:["key"] + <-Map 4 [SIMPLE_EDGE] + SHUFFLE [RS_12] + PartitionCols:_col0 + Select Operator [SEL_10] (rows=1 width=0) + Output:["_col0"] + Filter Operator [FIL_19] (rows=1 width=0) + predicate:key is not null + TableScan [TS_8] (rows=1 width=0) + default@q1,a,Tbl:PARTIAL,Col:NONE,Output:["key"] + Stage-2 + Dependency Collection{} + Stage-1 + Map 1 + File Output Operator [FS_3] + table:{"name:":"default.q1"} + Select Operator [SEL_2] (rows=250 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_4] (rows=250 width=10) + predicate:(key = '5') + TableScan [TS_0] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Stage-0 + Move Operator + Please refer to the previous Stage-1 + +PREHOOK: query: with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as (select * from src where key= '5') +select a.key +from q1 a join q1 b +on a.key=b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 +5 +5 +5 +5 +5 +5 +5 +5 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +values__tmp__table__1 diff --git ql/src/test/results/clientpositive/tez/cte_mat_5.q.out ql/src/test/results/clientpositive/tez/cte_mat_5.q.out new file mode 100644 index 0000000..52342c5 --- /dev/null +++ ql/src/test/results/clientpositive/tez/cte_mat_5.q.out @@ -0,0 +1,149 @@ +PREHOOK: query: create database mydb +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:mydb +POSTHOOK: query: create database mydb +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:mydb +PREHOOK: query: use mydb +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:mydb +POSTHOOK: query: use mydb +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:mydb +PREHOOK: query: create table q1 (colnum int, colstring string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:mydb +PREHOOK: Output: mydb@q1 +POSTHOOK: query: create table q1 (colnum int, colstring string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:mydb +POSTHOOK: Output: mydb@q1 +PREHOOK: query: insert into q1 values (5, 'A') +PREHOOK: type: QUERY +PREHOOK: Input: mydb@values__tmp__table__1 +PREHOOK: Output: mydb@q1 +POSTHOOK: query: insert into q1 values (5, 'A') +POSTHOOK: type: QUERY +POSTHOOK: Input: mydb@values__tmp__table__1 +POSTHOOK: Output: mydb@q1 +POSTHOOK: Lineage: q1.colnum EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: q1.colstring SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: use default +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:default +POSTHOOK: query: use default +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:default +PREHOOK: query: show tables in mydb +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:mydb +POSTHOOK: query: show tables in mydb +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:mydb +q1 +values__tmp__table__1 +PREHOOK: query: show tables +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: show tables +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +alltypesorc +cbo_t1 +cbo_t2 +cbo_t3 +lineitem +part +src +src1 +src_cbo +src_json +src_sequencefile +src_thrift +srcbucket +srcbucket2 +srcpart +PREHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + +Stage-3 + Fetch Operator + limit:-1 + Stage-4 + Reducer 3 + File Output Operator [FS_15] + Merge Join Operator [MERGEJOIN_20] (rows=1 width=3) + Conds:RS_11.UDFToDouble(_col0)=RS_12.UDFToDouble(_col0)(Inner),Output:["_col0"] + <-Map 2 [SIMPLE_EDGE] + SHUFFLE [RS_11] + PartitionCols:UDFToDouble(_col0) + Select Operator [SEL_7] (rows=1 width=3) + Output:["_col0"] + Filter Operator [FIL_18] (rows=1 width=3) + predicate:colnum is not null + TableScan [TS_5] (rows=1 width=3) + mydb@q1,a,Tbl:COMPLETE,Col:NONE,Output:["colnum"] + <-Map 4 [SIMPLE_EDGE] + SHUFFLE [RS_12] + PartitionCols:UDFToDouble(_col0) + Select Operator [SEL_10] (rows=1 width=0) + Output:["_col0"] + Filter Operator [FIL_19] (rows=1 width=0) + predicate:key is not null + TableScan [TS_8] (rows=1 width=0) + default@q1,b,Tbl:PARTIAL,Col:NONE,Output:["key"] + Stage-2 + Dependency Collection{} + Stage-1 + Map 1 + File Output Operator [FS_3] + table:{"name:":"default.q1"} + Select Operator [SEL_2] (rows=250 width=10) + Output:["_col0","_col1"] + Filter Operator [FIL_4] (rows=250 width=10) + predicate:(key = '5') + TableScan [TS_0] (rows=500 width=10) + default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Stage-0 + Move Operator + Please refer to the previous Stage-1 + +PREHOOK: query: with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@q1 +PREHOOK: Input: default@src +PREHOOK: Input: mydb@q1 +PREHOOK: Output: database:default +PREHOOK: Output: default@q1 +#### A masked pattern was here #### +POSTHOOK: query: with q1 as (select * from src where key= '5') +select a.colnum +from mydb.q1 as a join q1 as b +on a.colnum=b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@q1 +POSTHOOK: Input: default@src +POSTHOOK: Input: mydb@q1 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@q1 +#### A masked pattern was here #### +5 +5 +5 diff --git ql/src/test/results/clientpositive/tez/tez_union.q.out ql/src/test/results/clientpositive/tez/tez_union.q.out index 8baa5cf..0fec89a 100644 --- ql/src/test/results/clientpositive/tez/tez_union.q.out +++ ql/src/test/results/clientpositive/tez/tez_union.q.out @@ -1405,7 +1405,6 @@ POSTHOOK: Input: default@table1 POSTHOOK: Input: default@table2 POSTHOOK: Output: database:default POSTHOOK: Output: default@TABLE3 -POSTHOOK: Output: default@table3 PREHOOK: query: explain formatted select count(*) from TABLE3 PREHOOK: type: QUERY POSTHOOK: query: explain formatted select count(*) from TABLE3