diff --git data/files/struct3_a.txt data/files/struct3_a.txt deleted file mode 100644 index 19dbd7f..0000000 --- data/files/struct3_a.txt +++ /dev/null @@ -1,2 +0,0 @@ -1|true,200,72909,3244222,-99999999999|original -2|0,100,483777,14,-23866739993|original \ No newline at end of file diff --git data/files/struct3_b.txt data/files/struct3_b.txt deleted file mode 100644 index 030e0c0..0000000 --- data/files/struct3_b.txt +++ /dev/null @@ -1 +0,0 @@ -3|true,400,44388,-100,953967041.,62.079153,718.78,1,verdict,verdict,timestamp,date,binary|new \ No newline at end of file diff --git data/files/struct3_c.txt data/files/struct3_c.txt deleted file mode 100644 index 236694b..0000000 --- data/files/struct3_c.txt +++ /dev/null @@ -1 +0,0 @@ -4|false,-67,833,63993,1255178165.77663,905070.974,-4314.7918,-1240033819,trial,trial,2016-03-0703:02:22.0,2016-03-07,binary|new \ No newline at end of file diff --git data/files/struct5_a.txt data/files/struct5_a.txt new file mode 100644 index 0000000..3ed20d5 --- /dev/null +++ data/files/struct5_a.txt @@ -0,0 +1,2 @@ +1|original|true,200,72909,3244222,-99999999999|after_original +2|original|0,100,483777,14,-23866739993|after_original \ No newline at end of file diff --git data/files/struct5_b.txt data/files/struct5_b.txt new file mode 100644 index 0000000..e4df3f5 --- /dev/null +++ data/files/struct5_b.txt @@ -0,0 +1 @@ +3|new|true,400,44388,-100,953967041.,62.079153,718.78,1,verdict,verdict,1987-02-21 19:48:29.0|after_new \ No newline at end of file diff --git data/files/struct5_c.txt data/files/struct5_c.txt new file mode 100644 index 0000000..0f3c79c --- /dev/null +++ data/files/struct5_c.txt @@ -0,0 +1 @@ +4|new|false,-67,833,63993,1255178165.77663,905070.974,-4314.7918,-1240033819,trial,trial,2016-03-07 03:02:22.0|after_new \ No newline at end of file diff --git llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java index 1befba7..4784524 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.List; +import org.apache.derby.catalog.TypeDescriptor; import org.apache.hadoop.hive.llap.counters.LlapIOCounters; import org.apache.hadoop.hive.llap.metrics.LlapDaemonIOMetrics; import org.apache.orc.impl.DataReaderProperties; @@ -60,6 +61,8 @@ import org.apache.hadoop.hive.ql.io.HdfsUtils; import org.apache.orc.CompressionKind; import org.apache.orc.DataReader; +import org.apache.orc.OrcUtils; +import org.apache.orc.TypeDescription; import org.apache.hadoop.hive.ql.io.orc.OrcFile; import org.apache.hadoop.hive.ql.io.orc.OrcFile.ReaderOptions; import org.apache.orc.OrcConf; @@ -144,6 +147,7 @@ public void resetBeforeOffer(OrcEncodedColumnBatch t) { // Read state. private int stripeIxFrom; private OrcFileMetadata fileMetadata; + private TypeDescription schema; private Path path; private Reader orcReader; private DataReader metadataReader; @@ -232,6 +236,7 @@ protected Void performDataRead() throws IOException { fileMetadata = getOrReadFileMetadata(); consumer.setFileMetadata(fileMetadata); validateFileMetadata(); + schema = OrcUtils.convertTypeFromProtobuf(fileMetadata.getTypes(), 0); if (columnIds == null) { columnIds = createColumnIds(fileMetadata); } @@ -257,7 +262,9 @@ protected Void performDataRead() throws IOException { boolean[] globalIncludes = null; boolean[] sargColumns = null; try { - globalIncludes = OrcInputFormat.genIncludedColumns(fileMetadata.getTypes(), columnIds, true); + // UNDONE: Currently, LLAP doesn't support Schema Evolution yet, so we always pass + // UNDONE: the file schema as the reader schema. + globalIncludes = OrcInputFormat.genIncludedColumns(schema, columnIds); if (sarg != null && stride != 0) { // TODO: move this to a common method int[] filterColumns = RecordReaderImpl.mapSargColumnsToOrcInternalColIdx( @@ -352,7 +359,9 @@ protected Void performDataRead() throws IOException { } else { // We are reading subset of the original columns, remove unnecessary bitmasks/etc. // This will never happen w/o high-level cache. - stripeIncludes = OrcInputFormat.genIncludedColumns(fileMetadata.getTypes(), cols, true); + // UNDONE: Currently, LLAP doesn't support Schema Evolution yet, so we always pass + // UNDONE: the file schema as the reader schema. + stripeIncludes = OrcInputFormat.genIncludedColumns(schema, cols); colRgs = genStripeColRgs(cols, colRgs); } @@ -700,11 +709,12 @@ private boolean determineRgsToRead(boolean[] globalIncludes, int rowIndexStride, ArrayList metadata) throws IOException { RecordReaderImpl.SargApplier sargApp = null; if (sarg != null && rowIndexStride != 0) { - List types = fileMetadata.getTypes(); + // UNDONE: Currently, LLAP doesn't support Schema Evolution yet, so we always pass + // UNDONE: the file schema as the reader schema. String[] colNamesForSarg = OrcInputFormat.getSargColumnNames( - columnNames, types, globalIncludes, fileMetadata.isOriginalFormat()); + columnNames, schema, globalIncludes); sargApp = new RecordReaderImpl.SargApplier(sarg, colNamesForSarg, - rowIndexStride, types, globalIncludes.length); + rowIndexStride, globalIncludes.length); } boolean hasAnyData = false; // readState should have been initialized by this time with an empty array. diff --git orc/src/java/org/apache/orc/OrcUtils.java orc/src/java/org/apache/orc/OrcUtils.java index dc83b9c..a4bfec4 100644 --- orc/src/java/org/apache/orc/OrcUtils.java +++ orc/src/java/org/apache/orc/OrcUtils.java @@ -246,7 +246,7 @@ public static void appendOrcTypesRebuildSubtypes(List result, { // Make room for MAP type. result.add(null); - + // Add MAP type pair in order to determine their subtype values. appendOrcTypesRebuildSubtypes(result, children.get(0)); int subtype2 = result.size(); @@ -384,7 +384,7 @@ public static int appendOrcTypesRebuildSubtypes(List result, { // Make room for MAP type. result.add(null); - + // Add MAP type pair in order to determine their subtype values. columnId = appendOrcTypesRebuildSubtypes(result, types, columnId); int subtype2 = result.size(); @@ -538,4 +538,53 @@ TypeDescription convertTypeFromProtobuf(List types, } return result; } + + /** + * Determine whether a schema has the ACID struct by examining whether the transaction related + * column names are present. + * @param type + * @return + */ + public static boolean checkAcidSchema(TypeDescription type) { + if (type.getCategory().equals(TypeDescription.Category.STRUCT)) { + List rootFields = type.getFieldNames(); + if (acidEventFieldNames.equals(rootFields)) { + return true; + } + } + return false; + } + + /** + * @param typeDescr + * @return ORC types for the ACID event based on the row's type description + */ + public static TypeDescription createEventSchema(TypeDescription typeDescr) { + TypeDescription result = TypeDescription.createStruct() + .addField("operation", TypeDescription.createInt()) + .addField("originalTransaction", TypeDescription.createLong()) + .addField("bucket", TypeDescription.createInt()) + .addField("rowId", TypeDescription.createLong()) + .addField("currentTransaction", TypeDescription.createLong()) + .addField("row", typeDescr.clone()); + return result; + } + + public static int ROW_CHILD = 5; + + public static TypeDescription createRowSchema(TypeDescription typeDescr) { + TypeDescription rowSchema = typeDescr.getChildren().get(ROW_CHILD); + // Clone so the ids get reset. + return rowSchema.clone(); + } + + public static final List acidEventFieldNames= new ArrayList(); + static { + acidEventFieldNames.add("operation"); + acidEventFieldNames.add("originalTransaction"); + acidEventFieldNames.add("bucket"); + acidEventFieldNames.add("rowId"); + acidEventFieldNames.add("currentTransaction"); + acidEventFieldNames.add("row"); + } } diff --git orc/src/java/org/apache/orc/TypeDescription.java orc/src/java/org/apache/orc/TypeDescription.java index ffe3c1f..edc9547 100644 --- orc/src/java/org/apache/orc/TypeDescription.java +++ orc/src/java/org/apache/orc/TypeDescription.java @@ -482,6 +482,49 @@ private TypeDescription(Category category) { } } + public static Category getCategoryFromTypeKind(OrcProto.Type.Kind kind) { + switch (kind) { + case BOOLEAN: + return Category.BOOLEAN; + case BYTE: + return Category.BYTE; + case SHORT: + return Category.SHORT; + case INT: + return Category.INT; + case LONG: + return Category.LONG; + case FLOAT: + return Category.FLOAT; + case DOUBLE: + return Category.DOUBLE; + case STRING: + return Category.STRING; + case BINARY: + return Category.BINARY; + case TIMESTAMP: + return Category.TIMESTAMP; + case LIST: + return Category.LIST; + case MAP: + return Category.MAP; + case STRUCT: + return Category.STRUCT; + case UNION: + return Category.UNION; + case DECIMAL: + return Category.DECIMAL; + case DATE: + return Category.DATE; + case VARCHAR: + return Category.VARCHAR; + case CHAR: + return Category.CHAR; + default: + throw new RuntimeException("Unexpected ORC data type kind " + kind.name()); + } + } + private int id = -1; private int maxId = -1; private TypeDescription parent; diff --git orc/src/java/org/apache/orc/impl/ReaderImpl.java orc/src/java/org/apache/orc/impl/ReaderImpl.java index d6df7d7..bdfe542 100644 --- orc/src/java/org/apache/orc/impl/ReaderImpl.java +++ orc/src/java/org/apache/orc/impl/ReaderImpl.java @@ -572,13 +572,6 @@ public RecordReader rows() throws IOException { @Override public RecordReader rows(Options options) throws IOException { LOG.info("Reading ORC rows from " + path + " with " + options); - boolean[] include = options.getInclude(); - // if included columns is null, then include all columns - if (include == null) { - include = new boolean[types.size()]; - Arrays.fill(include, true); - options.include(include); - } return new RecordReaderImpl(this, options); } @@ -608,18 +601,28 @@ public static long getRawDataSizeFromColIndices( List stats) { long result = 0; for (int colIdx : colIndices) { - result += getRawDataSizeOfColumn(colIdx, types, stats); + TypeDescription.Category category = + TypeDescription.getCategoryFromTypeKind(types.get(colIdx).getKind()); + result += getRawDataSizeOfColumn(colIdx, category, stats); } return result; } - private static long getRawDataSizeOfColumn(int colIdx, List types, + public static long getRawDataSizeFromFileTypeDescrs(TypeDescription[] fileTypeDescrs, + List stats) { + long result = 0; + for (TypeDescription fileTypeDescr : fileTypeDescrs) { + result += getRawDataSizeOfColumn(fileTypeDescr.getId(), fileTypeDescr.getCategory(), stats); + } + return result; + } + + private static long getRawDataSizeOfColumn(int colIdx, TypeDescription.Category category, List stats) { OrcProto.ColumnStatistics colStat = stats.get(colIdx); long numVals = colStat.getNumberOfValues(); - OrcProto.Type type = types.get(colIdx); - switch (type.getKind()) { + switch (category) { case BINARY: // old orc format doesn't support binary statistics. checking for binary // statistics is not required as protocol buffers takes care of it. @@ -650,8 +653,15 @@ private static long getRawDataSizeOfColumn(int colIdx, List types case BOOLEAN: case BYTE: return numVals * JavaDataModel.get().primitive1(); + + case LIST: + case MAP: + case STRUCT: + case UNION: + return 0; + default: - LOG.debug("Unknown primitive category: " + type.getKind()); + LOG.debug("Unknown ORC category: " + category); break; } diff --git orc/src/java/org/apache/orc/impl/RecordReaderImpl.java orc/src/java/org/apache/orc/impl/RecordReaderImpl.java index 36a802e..6038ded 100644 --- orc/src/java/org/apache/orc/impl/RecordReaderImpl.java +++ orc/src/java/org/apache/orc/impl/RecordReaderImpl.java @@ -134,24 +134,36 @@ static int findColumns(String[] columnNames, protected RecordReaderImpl(ReaderImpl fileReader, Reader.Options options) throws IOException { - SchemaEvolution treeReaderSchema; - this.included = options.getInclude(); - included[0] = true; + if (options.getInclude() == null) { + // When reading ACID tables, we always have options.getSchema(). So no need to strip off + // ACID columns. + TypeDescription includeDeterminationSchema = + (options.getSchema() != null ? options.getSchema() : fileReader.getSchema()); + int includedSize = includeDeterminationSchema.getMaximumId() + 1; + this.included = new boolean[includedSize]; + Arrays.fill(this.included, true); + } else { + // Always include outer row struct (element 0). + this.included = options.getInclude(); + included[0] = true; + } + + SchemaEvolution evolution; if (options.getSchema() == null) { if (LOG.isInfoEnabled()) { LOG.info("Schema on read not provided -- using file schema " + fileReader.getSchema()); } - treeReaderSchema = new SchemaEvolution(fileReader.getSchema(), included); + evolution = new SchemaEvolution(fileReader.getSchema(), included); } else { // Now that we are creating a record reader for a file, validate that the schema to read // is compatible with the file schema. // - treeReaderSchema = new SchemaEvolution(fileReader.getSchema(), - options.getSchema(),included); + evolution = new SchemaEvolution(fileReader.getSchema(), options.getSchema(), included); } - this.schema = treeReaderSchema.getReaderSchema(); + + this.schema = evolution.getReaderSchema(); this.path = fileReader.path; this.codec = fileReader.codec; this.types = fileReader.types; @@ -160,7 +172,7 @@ protected RecordReaderImpl(ReaderImpl fileReader, SearchArgument sarg = options.getSearchArgument(); if (sarg != null && rowIndexStride != 0) { sargApp = new SargApplier( - sarg, options.getColumnNames(), rowIndexStride, types, + sarg, options.getColumnNames(), rowIndexStride, included.length); } else { sargApp = null; @@ -205,8 +217,8 @@ protected RecordReaderImpl(ReaderImpl fileReader, skipCorrupt = OrcConf.SKIP_CORRUPT_DATA.getBoolean(fileReader.conf); } - reader = TreeReaderFactory.createTreeReader(treeReaderSchema.getReaderSchema(), - treeReaderSchema, included, skipCorrupt); + reader = TreeReaderFactory.createTreeReader(evolution.getReaderSchema(), + evolution, included, skipCorrupt); indexes = new OrcProto.RowIndex[types.size()]; bloomFilterIndices = new OrcProto.BloomFilterIndex[types.size()]; advanceToNextRow(reader, 0L, true); @@ -707,7 +719,7 @@ private static Object getBaseObjectForComparison(PredicateLeaf.Type type, Object private final boolean[] sargColumns; public SargApplier(SearchArgument sarg, String[] columnNames, long rowIndexStride, - List types, int includedCount) { + int includedCount) { this.sarg = sarg; sargLeaves = sarg.getLeaves(); filterColumns = mapSargColumnsToOrcInternalColIdx(sargLeaves, columnNames, 0); diff --git orc/src/java/org/apache/orc/impl/SchemaEvolution.java orc/src/java/org/apache/orc/impl/SchemaEvolution.java index a6c1d60..ea8106a 100644 --- orc/src/java/org/apache/orc/impl/SchemaEvolution.java +++ orc/src/java/org/apache/orc/impl/SchemaEvolution.java @@ -20,14 +20,18 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.orc.OrcUtils; import org.apache.orc.TypeDescription; +import com.google.common.base.Preconditions; + /** * Take the file types and the (optional) configuration column names/types and see if there * has been schema evolution. @@ -35,11 +39,18 @@ public class SchemaEvolution { private final Map readerToFile; private final boolean[] included; + private final int includeFirstId; + private final int includeLastId; + private TypeDescription[] fileIncludedTypeDescrs; private final TypeDescription readerSchema; private static final Log LOG = LogFactory.getLog(SchemaEvolution.class); public SchemaEvolution(TypeDescription readerSchema, boolean[] included) { + Preconditions.checkState(!OrcUtils.checkAcidSchema(readerSchema)); this.included = included; + includeFirstId = 0; + includeLastId = readerSchema.getMaximumId() - 1; + fileIncludedTypeDescrs = null; readerToFile = null; this.readerSchema = readerSchema; } @@ -49,11 +60,25 @@ public SchemaEvolution(TypeDescription fileSchema, boolean[] included) throws IOException { readerToFile = new HashMap<>(readerSchema.getMaximumId() + 1); this.included = included; - if (checkAcidSchema(fileSchema)) { - this.readerSchema = createEventSchema(readerSchema); + fileIncludedTypeDescrs = null; + if (OrcUtils.checkAcidSchema(fileSchema)) { + this.readerSchema = OrcUtils.createEventSchema(readerSchema); + + /* + * The included array applies to the row only. + */ + TypeDescription rowType = this.readerSchema.getChildren().get(OrcUtils.ROW_CHILD); + includeFirstId = rowType.getId(); + includeLastId = rowType.getMaximumId() - 1; } else { this.readerSchema = readerSchema; + includeFirstId = 0; + includeLastId = readerSchema.getMaximumId() - 1; } + + // Make sure all the ids are assigned. + fileSchema.getId(); + buildMapping(fileSchema, this.readerSchema); } @@ -64,10 +89,15 @@ public TypeDescription getReaderSchema() { public TypeDescription getFileType(TypeDescription readerType) { TypeDescription result; if (readerToFile == null) { - if (included == null || included[readerType.getId()]) { + int id = readerType.getId(); + if (included == null || id < includeFirstId || id > includeLastId) { result = readerType; } else { - result = null; + if (included[id - includeFirstId]) { + result = readerType; + } else { + result = null; + } } } else { result = readerToFile.get(readerType); @@ -75,11 +105,33 @@ public TypeDescription getFileType(TypeDescription readerType) { return result; } + public int getIncludeFirstId() { + return includeFirstId; + } + + public int getIncludeLastId() { + return includeLastId; + } + + public TypeDescription[] getIncludedFileTypeDescrs() { + if (fileIncludedTypeDescrs == null) { + if (readerToFile == null) { + fileIncludedTypeDescrs = flattenFileIncluded(); + } else { + fileIncludedTypeDescrs = flattenReaderIncluded(); + } + } + return fileIncludedTypeDescrs; + } + void buildMapping(TypeDescription fileType, TypeDescription readerType) throws IOException { // if the column isn't included, don't map it - if (included != null && !included[readerType.getId()]) { - return; + int id = readerType.getId(); + if (included != null && id >= includeFirstId && id <= includeLastId) { + if (!included[id - includeFirstId]) { + return; + } } boolean isOk = true; // check the easy case first @@ -151,38 +203,44 @@ void buildMapping(TypeDescription fileType, } } - private static boolean checkAcidSchema(TypeDescription type) { - if (type.getCategory().equals(TypeDescription.Category.STRUCT)) { - List rootFields = type.getFieldNames(); - if (acidEventFieldNames.equals(rootFields)) { - return true; + TypeDescription[] flattenFileIncluded() { + List fileIncludedTypeList = new ArrayList(); + flattenFileIncludedRecurse(readerSchema, included, fileIncludedTypeList); + return fileIncludedTypeList.toArray(new TypeDescription[0]); + } + + void flattenFileIncludedRecurse(TypeDescription schema, boolean[] included, + List fileIncludedTypeList) { + int id = schema.getId(); + if (included == null || included[id]) { + fileIncludedTypeList.add(schema); + } + List children = schema.getChildren(); + if (children != null) { + for (TypeDescription child : children) { + flattenFileIncludedRecurse(child, included, fileIncludedTypeList); } } - return false; } - /** - * @param typeDescr - * @return ORC types for the ACID event based on the row's type description - */ - public static TypeDescription createEventSchema(TypeDescription typeDescr) { - TypeDescription result = TypeDescription.createStruct() - .addField("operation", TypeDescription.createInt()) - .addField("originalTransaction", TypeDescription.createLong()) - .addField("bucket", TypeDescription.createInt()) - .addField("rowId", TypeDescription.createLong()) - .addField("currentTransaction", TypeDescription.createLong()) - .addField("row", typeDescr.clone()); - return result; + TypeDescription[] flattenReaderIncluded() { + List fileIncludedIdList = new ArrayList(); + flattenFileIncludedRecurse(readerSchema, included, fileIncludedIdList); + return fileIncludedIdList.toArray(new TypeDescription[0]); } - public static final List acidEventFieldNames= new ArrayList(); - static { - acidEventFieldNames.add("operation"); - acidEventFieldNames.add("originalTransaction"); - acidEventFieldNames.add("bucket"); - acidEventFieldNames.add("rowId"); - acidEventFieldNames.add("currentTransaction"); - acidEventFieldNames.add("row"); + void flattenReaderIncludedRecurse(TypeDescription schema, boolean[] included, + List fileIncludedTypeList) { + TypeDescription fileSchema = readerToFile.get(schema); + if (fileSchema == null) { + return; + } + fileIncludedTypeList.add(fileSchema); + List children = schema.getChildren(); + if (children != null) { + for (TypeDescription child : children) { + flattenFileIncludedRecurse(child, included, fileIncludedTypeList); + } + } } } diff --git orc/src/java/org/apache/orc/impl/TreeReaderFactory.java orc/src/java/org/apache/orc/impl/TreeReaderFactory.java index c4a2093..4d816c3 100644 --- orc/src/java/org/apache/orc/impl/TreeReaderFactory.java +++ orc/src/java/org/apache/orc/impl/TreeReaderFactory.java @@ -2032,12 +2032,19 @@ public static TreeReader createTreeReader(TypeDescription readerType, boolean skipCorrupt ) throws IOException { TypeDescription fileType = evolution.getFileType(readerType); - if (fileType == null || - (included != null && !included[readerType.getId()])) { + if (fileType == null) { return new NullTreeReader(0); } + if (included != null) { + int includeFirstId = evolution.getIncludeFirstId(); + int includeLastId = evolution.getIncludeLastId(); + int id = readerType.getId(); + if (id >= includeFirstId && id <= includeLastId && !included[id - includeFirstId]) { + return new NullTreeReader(0); + } + } TypeDescription.Category readerTypeCategory = readerType.getCategory(); - if (!fileType.equals(readerType) && + if (fileType.getCategory() != readerTypeCategory && (readerTypeCategory != TypeDescription.Category.STRUCT && readerTypeCategory != TypeDescription.Category.MAP && readerTypeCategory != TypeDescription.Category.LIST && diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index 69d58d6..24cbc62 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.io.orc; import org.apache.orc.impl.InStream; +import org.apache.orc.impl.SchemaEvolution; import java.io.IOException; import java.nio.ByteBuffer; @@ -116,6 +117,7 @@ import org.apache.orc.OrcProto; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.protobuf.CodedInputStream; @@ -310,10 +312,20 @@ public static RecordReader createReaderFromFile(Reader file, Reader.Options options = new Reader.Options().range(offset, length); options.schema(schema); - boolean isOriginal = isOriginal(file); - List types = file.getTypes(); - options.include(genIncludedColumns(types, conf, isOriginal)); - setSearchArgument(options, types, conf, isOriginal); + + // We need the reader schema for genIncludedColumns / setSearchArgument. + TypeDescription readerSchema; + if (schema != null) { + readerSchema = schema; + } else { + readerSchema = file.getSchema(); + if (OrcUtils.checkAcidSchema(readerSchema)) { + readerSchema = OrcUtils.createRowSchema(readerSchema); + } + } + + options.include(genIncludedColumns(readerSchema, conf)); + setSearchArgument(options, readerSchema, conf); return (RecordReader) file.rowsOptions(options); } @@ -323,80 +335,103 @@ public static boolean isOriginal(Reader file) { /** * Recurse down into a type subtree turning on all of the sub-columns. - * @param types the types of the file + * + * NOTE: Currently, the included columns are just the top level columns. Any STRUCT type columns + * will have all of their columns included, recursively. There is open HIVE-13873 that would + * enhance things to have included columns have depth into STRUCT type columns so unnecessary + * STRUCT columns are not read. + * + * @param typeDescr the possible complex type. * @param result the global view of columns that should be included - * @param typeId the root of tree to enable - * @param rootColumn the top column */ - private static void includeColumnRecursive(List types, - boolean[] result, - int typeId, - int rootColumn) { - result[typeId - rootColumn] = true; - OrcProto.Type type = types.get(typeId); - int children = type.getSubtypesCount(); - for(int i=0; i < children; ++i) { - includeColumnRecursive(types, result, type.getSubtypes(i), rootColumn); + private static void includeColumnRecursive(TypeDescription typeDescr, + boolean[] result) { + result[typeDescr.getId()] = true; + final List children = typeDescr.getChildren(); + if (children == null) { + return; + } + final int numChildren = children.size(); + for(int i=0; i < numChildren; ++i) { + TypeDescription child = children.get(i); + includeColumnRecursive(child, result); } } - public static boolean[] genIncludedColumns( - List types, List included, boolean isOriginal) { - int rootColumn = getRootColumn(isOriginal); - int numColumns = types.size() - rootColumn; - boolean[] result = new boolean[numColumns]; + /** + * Generate an included boolean array from the reader schema and list of column ids. + * @param readerSchema + * @param included + * @return + */ + public static boolean[] genIncludedColumns(TypeDescription readerSchema, List included) { + if (OrcUtils.checkAcidSchema(readerSchema)) { + // Include column ids are for desired reader schema and do not include ACID columns. + readerSchema = OrcUtils.createRowSchema(readerSchema); + } + boolean[] result = new boolean[readerSchema.getMaximumId() + 1]; result[0] = true; - OrcProto.Type root = types.get(rootColumn); - for (int i = 0; i < root.getSubtypesCount(); ++i) { + final List children = readerSchema.getChildren(); + final int numChildren = children.size(); + for (int i = 0; i < numChildren; ++i) { + TypeDescription child = children.get(i); if (included.contains(i)) { - includeColumnRecursive(types, result, root.getSubtypes(i), rootColumn); + includeColumnRecursive(child, result); } } return result; } /** - * Take the configuration and figure out which columns we need to include. - * @param types the types for the file - * @param conf the configuration - * @param isOriginal is the file in the original format? + * Determine the logical included columns from the reader schema and configuration. + * + * @param readerSchema + * @param conf + * @return Will be null if included colummns are not present in the configuration. */ public static boolean[] genIncludedColumns( - List types, Configuration conf, boolean isOriginal) { - if (!ColumnProjectionUtils.isReadAllColumns(conf)) { + TypeDescription readerSchema, Configuration conf) { + if (OrcUtils.checkAcidSchema(readerSchema)) { + // Include column ids are for desired reader schema and do not include ACID columns. + readerSchema = OrcUtils.createRowSchema(readerSchema); + } + if (!ColumnProjectionUtils.isReadAllColumns(conf)) { List included = ColumnProjectionUtils.getReadColumnIDs(conf); - return genIncludedColumns(types, included, isOriginal); + return genIncludedColumns(readerSchema, included); } else { return null; } } public static String[] getSargColumnNames(String[] originalColumnNames, - List types, boolean[] includedColumns, boolean isOriginal) { - int rootColumn = getRootColumn(isOriginal); - String[] columnNames = new String[types.size() - rootColumn]; + TypeDescription readerSchema, boolean[] includedColumns) { + if (OrcUtils.checkAcidSchema(readerSchema)) { + // Include column ids are for desired reader schema and do not include ACID columns. + readerSchema = OrcUtils.createRowSchema(readerSchema); + } + String[] columnNames = new String[readerSchema.getMaximumId() + 1]; int i = 0; + final List children = readerSchema.getChildren(); // The way this works is as such. originalColumnNames is the equivalent on getNeededColumns // from TSOP. They are assumed to be in the same order as the columns in ORC file, AND they are // assumed to be equivalent to the columns in includedColumns (because it was generated from - // the same column list at some point in the past), minus the subtype columns. Therefore, when // we go thru all the top level ORC file columns that are included, in order, they match // originalColumnNames. This way, we do not depend on names stored inside ORC for SARG leaf // column name resolution (see mapSargColumns method). - for(int columnId: types.get(rootColumn).getSubtypesList()) { - if (includedColumns == null || includedColumns[columnId - rootColumn]) { + for(TypeDescription child : children) { + int id = child.getId(); + if (includedColumns == null || includedColumns[id]) { // this is guaranteed to be positive because types only have children // ids greater than their own id. - columnNames[columnId - rootColumn] = originalColumnNames[i++]; + columnNames[id] = originalColumnNames[i++]; } } return columnNames; } static void setSearchArgument(Reader.Options options, - List types, - Configuration conf, - boolean isOriginal) { + TypeDescription readerSchema, + Configuration conf) { String neededColumnNames = getNeededColumnNamesString(conf); if (neededColumnNames == null) { LOG.debug("No ORC pushdown predicate - no column names"); @@ -414,7 +449,7 @@ static void setSearchArgument(Reader.Options options, LOG.info("ORC pushdown predicate: " + sarg); } options.searchArgument(sarg, getSargColumnNames( - neededColumnNames.split(","), types, options.getInclude(), isOriginal)); + neededColumnNames.split(","), readerSchema, options.getInclude())); } static boolean canCreateSargFromConf(Configuration conf) { @@ -430,17 +465,17 @@ static boolean canCreateSargFromConf(Configuration conf) { } private static String[] extractNeededColNames( - List types, Configuration conf, boolean[] include, boolean isOriginal) { + TypeDescription readerSchema, Configuration conf, boolean[] include) { String colNames = getNeededColumnNamesString(conf); if (colNames == null) { return null; } - return extractNeededColNames(types, colNames, include, isOriginal); + return extractNeededColNames(readerSchema, colNames, include); } private static String[] extractNeededColNames( - List types, String columnNamesString, boolean[] include, boolean isOriginal) { - return getSargColumnNames(columnNamesString.split(","), types, include, isOriginal); + TypeDescription readerSchema, String columnNamesString, boolean[] include) { + return getSargColumnNames(columnNamesString.split(","), readerSchema, include); } static String getNeededColumnNamesString(Configuration conf) { @@ -1061,6 +1096,8 @@ private AcidDirInfo callInternal() throws IOException { private final long blockSize; private final TreeMap locations; private OrcTail orcTail; + private TypeDescription schema; + private SchemaEvolution evolution; private List stripes; private List stripeStats; private List types; @@ -1255,7 +1292,7 @@ public String toString() { // We can't eliminate stripes if there are deltas because the // deltas may change the rows making them match the predicate. if ((deltas == null || deltas.isEmpty()) && context.sarg != null) { - String[] colNames = extractNeededColNames(types, context.conf, includedCols, isOriginal); + String[] colNames = extractNeededColNames(evolution.getReaderSchema(), context.conf, includedCols); if (colNames == null) { LOG.warn("Skipping split elimination for {} as column names is null", file.getPath()); } else { @@ -1375,29 +1412,46 @@ private void populateAndCacheStripeDetails() throws IOException { stripes = orcTail.getStripes(); stripeStats = orcTail.getStripeStatistics(); types = orcTail.getTypes(); + schema = OrcUtils.convertTypeFromProtobuf(types, 0); writerVersion = orcTail.getWriterVersion(); - includedCols = genIncludedColumns(types, context.conf, isOriginal); + + boolean isAcidRead = HiveConf.getBoolVar(context.conf, ConfVars.HIVE_TRANSACTIONAL_TABLE_SCAN); + TypeDescription desiredSchema = + getDesiredRowTypeDescr(context.conf, isAcidRead, Integer.MAX_VALUE); + + // When reading ACID tables, we always have desiredSchema. So no need to strip off + // ACID columns. + Preconditions.checkState(desiredSchema != null || !isAcidRead); + + TypeDescription includeDeterminationSchema; + if (desiredSchema != null) { + includeDeterminationSchema = desiredSchema; + } else { + if (OrcUtils.checkAcidSchema(schema)) { + includeDeterminationSchema = OrcUtils.createRowSchema(schema); + } else { + includeDeterminationSchema = schema; + } + } + includedCols = genIncludedColumns(includeDeterminationSchema, context.conf); + + // Create the mapping from reader to file schema that we can pass it to + // getRawDataSizeFromFileTypeDescrs. It ignores ACID columns. + if (desiredSchema != null) { + evolution = new SchemaEvolution(desiredSchema, includeDeterminationSchema, includedCols); + } else { + evolution = new SchemaEvolution(includeDeterminationSchema, includedCols); + } + List fileColStats = orcTail.getFooter().getStatisticsList(); - projColsUncompressedSize = computeProjectionSize(types, fileColStats, includedCols, - isOriginal); + projColsUncompressedSize = + ReaderImpl.getRawDataSizeFromFileTypeDescrs( + evolution.getIncludedFileTypeDescrs(), fileColStats); + if (!context.footerInSplits) { orcTail = null; } } - - private long computeProjectionSize(List types, - List stats, boolean[] includedCols, boolean isOriginal) { - final int rootIdx = getRootColumn(isOriginal); - List internalColIds = Lists.newArrayList(); - if (includedCols != null) { - for (int i = 0; i < includedCols.length; i++) { - if (includedCols[i]) { - internalColIds.add(rootIdx + i); - } - } - } - return ReaderImpl.getRawDataSizeFromColIndices(internalColIds, types, stats); - } } @@ -1703,19 +1757,18 @@ public float getProgress() throws IOException { /** - * Do we have schema on read in the configuration variables? + * Get schema on read in the configuration variables for ACID. */ - TypeDescription schema = getDesiredRowTypeDescr(conf, true, Integer.MAX_VALUE); + final TypeDescription schema = getDesiredRowTypeDescr(conf, true, Integer.MAX_VALUE); + Preconditions.checkState(schema != null); // We always get a schema for ACID. final Reader reader; final int bucket; Reader.Options readOptions = new Reader.Options().schema(schema); readOptions.range(split.getStart(), split.getLength()); - // TODO: Convert genIncludedColumns and setSearchArgument to use TypeDescription. - final List schemaTypes = OrcUtils.getOrcTypes(schema); - readOptions.include(genIncludedColumns(schemaTypes, conf, SCHEMA_TYPES_IS_ORIGINAL)); - setSearchArgument(readOptions, schemaTypes, conf, SCHEMA_TYPES_IS_ORIGINAL); + readOptions.include(genIncludedColumns(schema, conf)); + setSearchArgument(readOptions, schema, conf); if (split.hasBase()) { bucket = AcidUtils.parseBaseBucketFilename(split.getPath(), conf) @@ -1741,7 +1794,7 @@ public float getProgress() throws IOException { @Override public ObjectInspector getObjectInspector() { - return OrcStruct.createObjectInspector(0, schemaTypes); + return OrcStruct.createObjectInspector(0, OrcUtils.getOrcTypes(schema)); } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java index 9bcdb39..4160e77 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java @@ -76,13 +76,6 @@ public RecordReader rows() throws IOException { @Override public RecordReader rowsOptions(Options options) throws IOException { LOG.info("Reading ORC rows from " + path + " with " + options); - boolean[] include = options.getInclude(); - // if included columns is null, then include all columns - if (include == null) { - include = new boolean[types.size()]; - Arrays.fill(include, true); - options.include(include); - } return new RecordReaderImpl(this, options); } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java index 9275aa9..3d177a2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java @@ -96,8 +96,8 @@ this.offset = fileSplit.getStart(); this.length = fileSplit.getLength(); options.range(offset, length); - options.include(OrcInputFormat.genIncludedColumns(types, conf, true)); - OrcInputFormat.setSearchArgument(options, types, conf, true); + options.include(OrcInputFormat.genIncludedColumns(schema, conf)); + OrcInputFormat.setSearchArgument(options, schema, conf); this.reader = file.rowsOptions(options); diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index e76c925..165017a 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -846,7 +846,7 @@ public void testInitiatorWithMultipleFailedCompactions() throws Exception { init.run(); int numAttemptedCompactions = 1; checkCompactionState(new CompactionsByState(numAttemptedCompactions,numFailedCompactions,0,0,0,0,numFailedCompactions + numAttemptedCompactions), countCompacts(txnHandler)); - + hiveConf.setTimeVar(HiveConf.ConfVars.COMPACTOR_HISTORY_REAPER_INTERVAL, 10, TimeUnit.MILLISECONDS); AcidCompactionHistoryService compactionHistoryService = new AcidCompactionHistoryService(); runHouseKeeperService(compactionHistoryService, hiveConf);//should not remove anything from history @@ -868,7 +868,7 @@ public void testInitiatorWithMultipleFailedCompactions() throws Exception { hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED), hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED),0,0,0,0, hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED) + hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED)), countCompacts(txnHandler)); - + hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILCOMPACTION, false); txnHandler.compact(new CompactionRequest("default", tblName, CompactionType.MINOR)); //at this point "show compactions" should have (COMPACTOR_HISTORY_RETENTION_FAILED) failed + 1 initiated (explicitly by user) @@ -1139,6 +1139,31 @@ public void testOpenTxnsCounter() throws Exception { Assert.assertNull(exception); } + @Test + public void testCompactWithDelete() throws Exception { + int[][] tableData = {{1,2},{3,4}}; + runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); + runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'"); + Worker t = new Worker(); + t.setThreadId((int) t.getId()); + t.setHiveConf(hiveConf); + AtomicBoolean stop = new AtomicBoolean(); + AtomicBoolean looped = new AtomicBoolean(); + stop.set(true); + t.init(stop, looped); + t.run(); + runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 4"); + runStatementOnDriver("update " + Table.ACIDTBL + " set b = -2 where b = 2"); + runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MINOR'"); + t.run(); + TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); + ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals("Unexpected number of compactions in history", 2, resp.getCompactsSize()); + Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(0).getState()); + Assert.assertEquals("Unexpected 1 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(1).getState()); + } + + /** * takes raw data and turns it into a string as if from Driver.getResults() * sorts rows in dictionary order diff --git ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java index 3c89dd6..a6fe4dd 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java @@ -115,6 +115,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Progressable; import org.apache.orc.OrcProto; +import org.apache.orc.OrcUtils; +import org.apache.orc.TypeDescription; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -2335,7 +2337,8 @@ public void testSetSearchArgument() throws Exception { conf.set(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, "url,cost"); options.include(new boolean[]{true, true, false, true, false}); - OrcInputFormat.setSearchArgument(options, types, conf, false); + TypeDescription readerSchema = OrcUtils.convertTypeFromProtobuf(types, OrcInputFormat.getRootColumn(false)); + OrcInputFormat.setSearchArgument(options, readerSchema, conf); String[] colNames = options.getColumnNames(); assertEquals(null, colNames[0]); assertEquals("url", colNames[1]); diff --git ql/src/test/queries/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q ql/src/test/queries/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q index 27cea8d..40172be 100644 --- ql/src/test/queries/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q +++ ql/src/test/queries/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q @@ -160,3 +160,46 @@ select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by in select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by insert_num; drop table part_add_to_various_various_struct4; + + + +-- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT); + +CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING); + +CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt; + +insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt; + +CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt; + +explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +drop table part_add_to_various_various_struct5; diff --git ql/src/test/queries/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q ql/src/test/queries/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q index f8a8fa6..1fd70a1 100644 --- ql/src/test/queries/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q +++ ql/src/test/queries/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q @@ -160,3 +160,46 @@ select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by in select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by insert_num; drop table part_add_to_various_various_struct4; + + + +-- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT); + +CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING); + +CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt; + +insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt; + +CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt; + +explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +drop table part_add_to_various_various_struct5; diff --git ql/src/test/queries/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q ql/src/test/queries/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q index c1e8af6..35767c1 100644 --- ql/src/test/queries/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q +++ ql/src/test/queries/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q @@ -160,3 +160,46 @@ select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by in select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by insert_num; drop table part_add_to_various_various_struct4; + + + +-- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT); + +CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING); + +CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt; + +insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt; + +CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt; + +explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +drop table part_add_to_various_various_struct5; diff --git ql/src/test/queries/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q ql/src/test/queries/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q index a914544..8272cd6 100644 --- ql/src/test/queries/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q +++ ql/src/test/queries/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q @@ -162,3 +162,46 @@ select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by in select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by insert_num; drop table part_add_to_various_various_struct4; + + + +-- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT); + +CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING); + +CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt; + +insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt; + +CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt; + +explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +drop table part_add_to_various_various_struct5; diff --git ql/src/test/queries/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q ql/src/test/queries/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q index 7737abf..babd032 100644 --- ql/src/test/queries/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q +++ ql/src/test/queries/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q @@ -163,3 +163,46 @@ select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by in select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by insert_num; drop table part_add_to_various_various_struct4; + + + +-- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT); + +CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING); + +CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt; + +insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt; + +CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt; + +explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +drop table part_add_to_various_various_struct5; diff --git ql/src/test/results/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out index 97e8fc0..bc2ff88 100644 --- ql/src/test/results/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out @@ -724,3 +724,216 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: part_add_to_various_various_struct5 + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: insert_num (type: int), part (type: int), b (type: string), s5 (type: struct), after_struct_column (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: struct), _col4 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: struct), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":905071.0,"c7":-4314.7918,"c8":-1240033819,"c9":"trial ","c10":"trial","c11":"2016-03-07 03:02:22"} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q.out index 62047fb..f8ce329 100644 --- ql/src/test/results/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q.out @@ -724,3 +724,216 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: part_add_to_various_various_struct5 + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: insert_num (type: int), part (type: int), b (type: string), s5 (type: struct), after_struct_column (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: struct), _col4 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: struct), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":905071.0,"c7":-4314.7918,"c8":-1240033819,"c9":"trial ","c10":"trial","c11":"2016-03-07 03:02:22"} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q.out index 2a5fb24..239b558 100644 --- ql/src/test/results/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q.out @@ -724,3 +724,216 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: part_add_to_various_various_struct5 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: insert_num (type: int), part (type: int), b (type: string), s5 (type: struct), after_struct_column (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: struct), _col4 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: struct), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q.out index 7a1ccfe..e03e2d5 100644 --- ql/src/test/results/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q.out @@ -728,3 +728,216 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: part_add_to_various_various_struct5 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: insert_num (type: int), part (type: int), b (type: string), s5 (type: struct), after_struct_column (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: struct), _col4 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: struct), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q.out index 8af1b4f..8f5e147 100644 --- ql/src/test/results/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q.out @@ -730,3 +730,216 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: part_add_to_various_various_struct5 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: insert_num (type: int), part (type: int), b (type: string), s5 (type: struct), after_struct_column (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: struct), _col4 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: struct), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/tez/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/tez/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out index 34795b1..c10a210 100644 --- ql/src/test/results/clientpositive/tez/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/tez/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out @@ -667,3 +667,197 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_4] + Select Operator [SEL_3] (rows=4 width=388) + Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=4 width=388) + Output:["_col0","_col1","_col2","_col3","_col4"] + TableScan [TS_0] (rows=4 width=388) + default@part_add_to_various_various_struct5,part_add_to_various_various_struct5,Tbl:COMPLETE,Col:NONE,Output:["insert_num","b","s5","after_struct_column"] + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":905071.0,"c7":-4314.7918,"c8":-1240033819,"c9":"trial ","c10":"trial","c11":"2016-03-07 03:02:22"} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/tez/schema_evol_orc_vec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/tez/schema_evol_orc_vec_mapwork_part_all_complex.q.out index c91eb0b..9aac540 100644 --- ql/src/test/results/clientpositive/tez/schema_evol_orc_vec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/tez/schema_evol_orc_vec_mapwork_part_all_complex.q.out @@ -667,3 +667,197 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_4] + Select Operator [SEL_3] (rows=4 width=388) + Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=4 width=388) + Output:["_col0","_col1","_col2","_col3","_col4"] + TableScan [TS_0] (rows=4 width=388) + default@part_add_to_various_various_struct5,part_add_to_various_various_struct5,Tbl:COMPLETE,Col:NONE,Output:["insert_num","b","s5","after_struct_column"] + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":905071.0,"c7":-4314.7918,"c8":-1240033819,"c9":"trial ","c10":"trial","c11":"2016-03-07 03:02:22"} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/tez/schema_evol_text_nonvec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/tez/schema_evol_text_nonvec_mapwork_part_all_complex.q.out index b0b00b8..49be99f 100644 --- ql/src/test/results/clientpositive/tez/schema_evol_text_nonvec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/tez/schema_evol_text_nonvec_mapwork_part_all_complex.q.out @@ -667,3 +667,197 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_4] + Select Operator [SEL_3] (rows=4 width=97) + Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=4 width=97) + Output:["_col0","_col1","_col2","_col3","_col4"] + TableScan [TS_0] (rows=4 width=97) + default@part_add_to_various_various_struct5,part_add_to_various_various_struct5,Tbl:COMPLETE,Col:NONE,Output:["insert_num","b","s5","after_struct_column"] + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/tez/schema_evol_text_vec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/tez/schema_evol_text_vec_mapwork_part_all_complex.q.out index 4e03c0b..cc7971d 100644 --- ql/src/test/results/clientpositive/tez/schema_evol_text_vec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/tez/schema_evol_text_vec_mapwork_part_all_complex.q.out @@ -671,3 +671,197 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_4] + Select Operator [SEL_3] (rows=4 width=97) + Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=4 width=97) + Output:["_col0","_col1","_col2","_col3","_col4"] + TableScan [TS_0] (rows=4 width=97) + default@part_add_to_various_various_struct5,part_add_to_various_various_struct5,Tbl:COMPLETE,Col:NONE,Output:["insert_num","b","s5","after_struct_column"] + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/tez/schema_evol_text_vecrow_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/tez/schema_evol_text_vecrow_mapwork_part_all_complex.q.out index f45b517..1312485 100644 --- ql/src/test/results/clientpositive/tez/schema_evol_text_vecrow_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/tez/schema_evol_text_vecrow_mapwork_part_all_complex.q.out @@ -673,3 +673,197 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_4] + Select Operator [SEL_3] (rows=4 width=97) + Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=4 width=97) + Output:["_col0","_col1","_col2","_col3","_col4"] + TableScan [TS_0] (rows=4 width=97) + default@part_add_to_various_various_struct5,part_add_to_various_various_struct5,Tbl:COMPLETE,Col:NONE,Output:["insert_num","b","s5","after_struct_column"] + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5