diff --git data/files/struct3_a.txt data/files/struct3_a.txt deleted file mode 100644 index 19dbd7f..0000000 --- data/files/struct3_a.txt +++ /dev/null @@ -1,2 +0,0 @@ -1|true,200,72909,3244222,-99999999999|original -2|0,100,483777,14,-23866739993|original \ No newline at end of file diff --git data/files/struct3_b.txt data/files/struct3_b.txt deleted file mode 100644 index 030e0c0..0000000 --- data/files/struct3_b.txt +++ /dev/null @@ -1 +0,0 @@ -3|true,400,44388,-100,953967041.,62.079153,718.78,1,verdict,verdict,timestamp,date,binary|new \ No newline at end of file diff --git data/files/struct3_c.txt data/files/struct3_c.txt deleted file mode 100644 index 236694b..0000000 --- data/files/struct3_c.txt +++ /dev/null @@ -1 +0,0 @@ -4|false,-67,833,63993,1255178165.77663,905070.974,-4314.7918,-1240033819,trial,trial,2016-03-0703:02:22.0,2016-03-07,binary|new \ No newline at end of file diff --git data/files/struct5_a.txt data/files/struct5_a.txt new file mode 100644 index 0000000..3ed20d5 --- /dev/null +++ data/files/struct5_a.txt @@ -0,0 +1,2 @@ +1|original|true,200,72909,3244222,-99999999999|after_original +2|original|0,100,483777,14,-23866739993|after_original \ No newline at end of file diff --git data/files/struct5_b.txt data/files/struct5_b.txt new file mode 100644 index 0000000..e4df3f5 --- /dev/null +++ data/files/struct5_b.txt @@ -0,0 +1 @@ +3|new|true,400,44388,-100,953967041.,62.079153,718.78,1,verdict,verdict,1987-02-21 19:48:29.0|after_new \ No newline at end of file diff --git data/files/struct5_c.txt data/files/struct5_c.txt new file mode 100644 index 0000000..0f3c79c --- /dev/null +++ data/files/struct5_c.txt @@ -0,0 +1 @@ +4|new|false,-67,833,63993,1255178165.77663,905070.974,-4314.7918,-1240033819,trial,trial,2016-03-07 03:02:22.0|after_new \ No newline at end of file diff --git orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java index c0c6964..99431ac 100644 --- orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java +++ orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java @@ -2066,7 +2066,6 @@ private static TreeReader createAnyIntegerConvertTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt) throws IOException { // CONVERT from (BOOLEAN, BYTE, SHORT, INT, LONG) to schema type. @@ -2123,7 +2122,6 @@ private static TreeReader createFloatConvertTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt) throws IOException { // CONVERT from FLOAT to schema type. @@ -2172,7 +2170,6 @@ private static TreeReader createDoubleConvertTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt) throws IOException { // CONVERT from DOUBLE to schema type. @@ -2221,7 +2218,6 @@ private static TreeReader createDecimalConvertTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt) throws IOException { // CONVERT from DECIMAL to schema type. @@ -2269,7 +2265,6 @@ private static TreeReader createStringConvertTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt) throws IOException { // CONVERT from STRING to schema type. @@ -2326,7 +2321,6 @@ private static TreeReader createCharConvertTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt) throws IOException { // CONVERT from CHAR to schema type. @@ -2382,7 +2376,6 @@ private static TreeReader createVarcharConvertTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt) throws IOException { // CONVERT from VARCHAR to schema type. @@ -2438,7 +2431,6 @@ private static TreeReader createTimestampConvertTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt) throws IOException { // CONVERT from TIMESTAMP to schema type. @@ -2489,7 +2481,6 @@ private static TreeReader createDateConvertTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt) throws IOException { // CONVERT from DATE to schema type. @@ -2532,7 +2523,6 @@ private static TreeReader createBinaryConvertTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt) throws IOException { // CONVERT from DATE to schema type. @@ -2691,14 +2681,12 @@ private static TreeReader createBinaryConvertTreeReader(int columnId, * * @param readerType * @param evolution - * @param included * @param skipCorrupt * @return * @throws IOException */ public static TreeReader createConvertTreeReader(TypeDescription readerType, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt ) throws IOException { @@ -2713,43 +2701,43 @@ public static TreeReader createConvertTreeReader(TypeDescription readerType, case INT: case LONG: return createAnyIntegerConvertTreeReader(columnId, fileType, readerType, evolution, - included, skipCorrupt); + skipCorrupt); case FLOAT: return createFloatConvertTreeReader(columnId, fileType, readerType, evolution, - included, skipCorrupt); + skipCorrupt); case DOUBLE: return createDoubleConvertTreeReader(columnId, fileType, readerType, evolution, - included, skipCorrupt); + skipCorrupt); case DECIMAL: return createDecimalConvertTreeReader(columnId, fileType, readerType, evolution, - included, skipCorrupt); + skipCorrupt); case STRING: return createStringConvertTreeReader(columnId, fileType, readerType, evolution, - included, skipCorrupt); + skipCorrupt); case CHAR: return createCharConvertTreeReader(columnId, fileType, readerType, evolution, - included, skipCorrupt); + skipCorrupt); case VARCHAR: return createVarcharConvertTreeReader(columnId, fileType, readerType, evolution, - included, skipCorrupt); + skipCorrupt); case TIMESTAMP: return createTimestampConvertTreeReader(columnId, fileType, readerType, evolution, - included, skipCorrupt); + skipCorrupt); case DATE: return createDateConvertTreeReader(columnId, fileType, readerType, evolution, - included, skipCorrupt); + skipCorrupt); case BINARY: return createBinaryConvertTreeReader(columnId, fileType, readerType, evolution, - included, skipCorrupt); + skipCorrupt); // UNDONE: Complex conversions... case STRUCT: diff --git orc/src/java/org/apache/orc/impl/RecordReaderImpl.java orc/src/java/org/apache/orc/impl/RecordReaderImpl.java index 23f532c..2e14718 100644 --- orc/src/java/org/apache/orc/impl/RecordReaderImpl.java +++ orc/src/java/org/apache/orc/impl/RecordReaderImpl.java @@ -206,7 +206,7 @@ protected RecordReaderImpl(ReaderImpl fileReader, } reader = TreeReaderFactory.createTreeReader(treeReaderSchema.getReaderSchema(), - treeReaderSchema, included, skipCorrupt); + treeReaderSchema, skipCorrupt); indexes = new OrcProto.RowIndex[types.size()]; bloomFilterIndices = new OrcProto.BloomFilterIndex[types.size()]; advanceToNextRow(reader, 0L, true); diff --git orc/src/java/org/apache/orc/impl/SchemaEvolution.java orc/src/java/org/apache/orc/impl/SchemaEvolution.java index a6c1d60..c0dc5a5 100644 --- orc/src/java/org/apache/orc/impl/SchemaEvolution.java +++ orc/src/java/org/apache/orc/impl/SchemaEvolution.java @@ -78,7 +78,11 @@ public TypeDescription getFileType(TypeDescription readerType) { void buildMapping(TypeDescription fileType, TypeDescription readerType) throws IOException { // if the column isn't included, don't map it - if (included != null && !included[readerType.getId()]) { + int typeId = readerType.getId(); + // UNDONE: Why, in somes cases is type id larger than included? + if (included != null && + typeId < included.length && + !included[readerType.getId()]) { return; } boolean isOk = true; diff --git orc/src/java/org/apache/orc/impl/TreeReaderFactory.java orc/src/java/org/apache/orc/impl/TreeReaderFactory.java index c4a2093..41da0d0 100644 --- orc/src/java/org/apache/orc/impl/TreeReaderFactory.java +++ orc/src/java/org/apache/orc/impl/TreeReaderFactory.java @@ -1705,7 +1705,6 @@ public void nextVector(ColumnVector previousVector, protected StructTreeReader(int columnId, TypeDescription readerSchema, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt) throws IOException { super(columnId); @@ -1713,7 +1712,7 @@ protected StructTreeReader(int columnId, this.fields = new TreeReader[childrenTypes.size()]; for (int i = 0; i < fields.length; ++i) { TypeDescription subtype = childrenTypes.get(i); - this.fields[i] = createTreeReader(subtype, evolution, included, skipCorrupt); + this.fields[i] = createTreeReader(subtype, evolution, skipCorrupt); } } @@ -1790,7 +1789,6 @@ void skipRows(long items) throws IOException { protected UnionTreeReader(int fileColumn, TypeDescription readerSchema, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt) throws IOException { super(fileColumn); List childrenTypes = readerSchema.getChildren(); @@ -1798,7 +1796,7 @@ protected UnionTreeReader(int fileColumn, this.fields = new TreeReader[fieldCount]; for (int i = 0; i < fieldCount; ++i) { TypeDescription subtype = childrenTypes.get(i); - this.fields[i] = createTreeReader(subtype, evolution, included, skipCorrupt); + this.fields[i] = createTreeReader(subtype, evolution, skipCorrupt); } } @@ -1867,12 +1865,10 @@ void skipRows(long items) throws IOException { protected ListTreeReader(int fileColumn, TypeDescription readerSchema, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt) throws IOException { super(fileColumn); TypeDescription elementType = readerSchema.getChildren().get(0); - elementReader = createTreeReader(elementType, evolution, included, - skipCorrupt); + elementReader = createTreeReader(elementType, evolution, skipCorrupt); } @Override @@ -1947,13 +1943,12 @@ void skipRows(long items) throws IOException { protected MapTreeReader(int fileColumn, TypeDescription readerSchema, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt) throws IOException { super(fileColumn); TypeDescription keyType = readerSchema.getChildren().get(0); TypeDescription valueType = readerSchema.getChildren().get(1); - keyReader = createTreeReader(keyType, evolution, included, skipCorrupt); - valueReader = createTreeReader(valueType, evolution, included, skipCorrupt); + keyReader = createTreeReader(keyType, evolution, skipCorrupt); + valueReader = createTreeReader(valueType, evolution, skipCorrupt); } @Override @@ -2026,25 +2021,57 @@ void skipRows(long items) throws IOException { } } + private static boolean isStructCategory(TypeDescription.Category typeCategory) { + return (typeCategory == TypeDescription.Category.STRUCT || + typeCategory == TypeDescription.Category.MAP || + typeCategory == TypeDescription.Category.LIST || + typeCategory == TypeDescription.Category.UNION); + } + + /** + * Determine whether primitive type conversion is needed based on the category and data type attributes + * maxLength and the decimal scale/precision. + * + * Field names are not part of this comparison. + * + * @param fileType + * @param readerType + * @return + */ + private static boolean primitiveTypeNeedsConversion(TypeDescription fileType, TypeDescription readerType) { + if (fileType.getCategory() != readerType.getCategory() || + fileType.getMaxLength() != readerType.getMaxLength() || + fileType.getScale() != readerType.getScale() || + fileType.getPrecision() != readerType.getPrecision()) { + return true; + } + return false; + } + public static TreeReader createTreeReader(TypeDescription readerType, SchemaEvolution evolution, - boolean[] included, boolean skipCorrupt ) throws IOException { + // The SchemaEvolution class has already processed the included (needed) columns. Here we + // just have to ask if there is a non-null file type. TypeDescription fileType = evolution.getFileType(readerType); - if (fileType == null || - (included != null && !included[readerType.getId()])) { + if (fileType == null) { return new NullTreeReader(0); } + TypeDescription.Category fileTypeCategory = fileType.getCategory(); TypeDescription.Category readerTypeCategory = readerType.getCategory(); - if (!fileType.equals(readerType) && - (readerTypeCategory != TypeDescription.Category.STRUCT && - readerTypeCategory != TypeDescription.Category.MAP && - readerTypeCategory != TypeDescription.Category.LIST && - readerTypeCategory != TypeDescription.Category.UNION)) { - // We only convert complex children. - return ConvertTreeReaderFactory.createConvertTreeReader(readerType, evolution, - included, skipCorrupt); + if (isStructCategory(fileTypeCategory) || isStructCategory(readerTypeCategory)) { + if (fileTypeCategory != readerTypeCategory) { + throw new IOException("Cannot convert between Complex file type " + + fileTypeCategory.name() + " and reader type " + readerTypeCategory.name()); + } + // Otherwise, fall through. The complex readers will automatically convert their children + // when the recursively call this method. + } else { + // Primitive types. + if (primitiveTypeNeedsConversion(fileType, readerType)) { + return ConvertTreeReaderFactory.createConvertTreeReader(readerType, evolution, skipCorrupt); + } } switch (readerTypeCategory) { case BOOLEAN: @@ -2077,17 +2104,13 @@ public static TreeReader createTreeReader(TypeDescription readerType, return new DecimalTreeReader(fileType.getId(), readerType.getPrecision(), readerType.getScale()); case STRUCT: - return new StructTreeReader(fileType.getId(), readerType, - evolution, included, skipCorrupt); + return new StructTreeReader(fileType.getId(), readerType, evolution, skipCorrupt); case LIST: - return new ListTreeReader(fileType.getId(), readerType, - evolution, included, skipCorrupt); + return new ListTreeReader(fileType.getId(), readerType, evolution, skipCorrupt); case MAP: - return new MapTreeReader(fileType.getId(), readerType, evolution, - included, skipCorrupt); + return new MapTreeReader(fileType.getId(), readerType, evolution, skipCorrupt); case UNION: - return new UnionTreeReader(fileType.getId(), readerType, - evolution, included, skipCorrupt); + return new UnionTreeReader(fileType.getId(), readerType, evolution, skipCorrupt); default: throw new IllegalArgumentException("Unsupported type " + readerTypeCategory); diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java index 9bcdb39..74c5071 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java @@ -79,6 +79,7 @@ public RecordReader rowsOptions(Options options) throws IOException { boolean[] include = options.getInclude(); // if included columns is null, then include all columns if (include == null) { + options = options.clone(); include = new boolean[types.size()]; Arrays.fill(include, true); options.include(include); diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index e76c925..245a3bc 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -846,7 +846,7 @@ public void testInitiatorWithMultipleFailedCompactions() throws Exception { init.run(); int numAttemptedCompactions = 1; checkCompactionState(new CompactionsByState(numAttemptedCompactions,numFailedCompactions,0,0,0,0,numFailedCompactions + numAttemptedCompactions), countCompacts(txnHandler)); - + hiveConf.setTimeVar(HiveConf.ConfVars.COMPACTOR_HISTORY_REAPER_INTERVAL, 10, TimeUnit.MILLISECONDS); AcidCompactionHistoryService compactionHistoryService = new AcidCompactionHistoryService(); runHouseKeeperService(compactionHistoryService, hiveConf);//should not remove anything from history @@ -868,7 +868,7 @@ public void testInitiatorWithMultipleFailedCompactions() throws Exception { hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED), hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED),0,0,0,0, hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED) + hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED)), countCompacts(txnHandler)); - + hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILCOMPACTION, false); txnHandler.compact(new CompactionRequest("default", tblName, CompactionType.MINOR)); //at this point "show compactions" should have (COMPACTOR_HISTORY_RETENTION_FAILED) failed + 1 initiated (explicitly by user) @@ -1139,6 +1139,30 @@ public void testOpenTxnsCounter() throws Exception { Assert.assertNull(exception); } + @Test + public void testCompactWithDelete() throws Exception { + int[][] tableData = {{1,2},{3,4}}; + runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData)); + runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'"); + Worker t = new Worker(); + t.setThreadId((int) t.getId()); + t.setHiveConf(hiveConf); + AtomicBoolean stop = new AtomicBoolean(); + AtomicBoolean looped = new AtomicBoolean(); + stop.set(true); + t.init(stop, looped); + t.run(); + runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 4"); + runStatementOnDriver("update " + Table.ACIDTBL + " set b = -2 where b = 2"); + runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MINOR'"); + t.run(); + TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf); + ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals("Unexpected number of compactions in history", 2, resp.getCompactsSize()); + Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(0).getState()); + Assert.assertEquals("Unexpected 1 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(1).getState()); + } + /** * takes raw data and turns it into a string as if from Driver.getResults() * sorts rows in dictionary order diff --git ql/src/test/queries/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q ql/src/test/queries/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q index 27cea8d..40172be 100644 --- ql/src/test/queries/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q +++ ql/src/test/queries/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q @@ -160,3 +160,46 @@ select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by in select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by insert_num; drop table part_add_to_various_various_struct4; + + + +-- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT); + +CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING); + +CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt; + +insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt; + +CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt; + +explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +drop table part_add_to_various_various_struct5; diff --git ql/src/test/queries/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q ql/src/test/queries/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q index f8a8fa6..1fd70a1 100644 --- ql/src/test/queries/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q +++ ql/src/test/queries/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q @@ -160,3 +160,46 @@ select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by in select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by insert_num; drop table part_add_to_various_various_struct4; + + + +-- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT); + +CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING); + +CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt; + +insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt; + +CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt; + +explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +drop table part_add_to_various_various_struct5; diff --git ql/src/test/queries/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q ql/src/test/queries/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q index c1e8af6..35767c1 100644 --- ql/src/test/queries/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q +++ ql/src/test/queries/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q @@ -160,3 +160,46 @@ select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by in select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by insert_num; drop table part_add_to_various_various_struct4; + + + +-- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT); + +CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING); + +CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt; + +insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt; + +CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt; + +explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +drop table part_add_to_various_various_struct5; diff --git ql/src/test/queries/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q ql/src/test/queries/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q index a914544..8272cd6 100644 --- ql/src/test/queries/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q +++ ql/src/test/queries/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q @@ -162,3 +162,46 @@ select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by in select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by insert_num; drop table part_add_to_various_various_struct4; + + + +-- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT); + +CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING); + +CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt; + +insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt; + +CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt; + +explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +drop table part_add_to_various_various_struct5; diff --git ql/src/test/queries/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q ql/src/test/queries/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q index 7737abf..babd032 100644 --- ql/src/test/queries/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q +++ ql/src/test/queries/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q @@ -163,3 +163,46 @@ select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by in select insert_num,part,b,s3 from part_add_to_various_various_struct4 order by insert_num; drop table part_add_to_various_various_struct4; + + + +-- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT); + +CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING); + +CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt; + +insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt; + +CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt; + +insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt; + +explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num; + +drop table part_add_to_various_various_struct5; diff --git ql/src/test/results/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out index 97e8fc0..bc2ff88 100644 --- ql/src/test/results/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out @@ -724,3 +724,216 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: part_add_to_various_various_struct5 + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: insert_num (type: int), part (type: int), b (type: string), s5 (type: struct), after_struct_column (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: struct), _col4 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: struct), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":905071.0,"c7":-4314.7918,"c8":-1240033819,"c9":"trial ","c10":"trial","c11":"2016-03-07 03:02:22"} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q.out index 62047fb..f8ce329 100644 --- ql/src/test/results/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/schema_evol_orc_vec_mapwork_part_all_complex.q.out @@ -724,3 +724,216 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: part_add_to_various_various_struct5 + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: insert_num (type: int), part (type: int), b (type: string), s5 (type: struct), after_struct_column (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: struct), _col4 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: struct), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 1554 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":905071.0,"c7":-4314.7918,"c8":-1240033819,"c9":"trial ","c10":"trial","c11":"2016-03-07 03:02:22"} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q.out index 2a5fb24..239b558 100644 --- ql/src/test/results/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/schema_evol_text_nonvec_mapwork_part_all_complex.q.out @@ -724,3 +724,216 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: part_add_to_various_various_struct5 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: insert_num (type: int), part (type: int), b (type: string), s5 (type: struct), after_struct_column (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: struct), _col4 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: struct), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q.out index 7a1ccfe..e03e2d5 100644 --- ql/src/test/results/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/schema_evol_text_vec_mapwork_part_all_complex.q.out @@ -728,3 +728,216 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: part_add_to_various_various_struct5 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: insert_num (type: int), part (type: int), b (type: string), s5 (type: struct), after_struct_column (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: struct), _col4 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: struct), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q.out index 8af1b4f..8f5e147 100644 --- ql/src/test/results/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/schema_evol_text_vecrow_mapwork_part_all_complex.q.out @@ -730,3 +730,216 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: part_add_to_various_various_struct5 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: insert_num (type: int), part (type: int), b (type: string), s5 (type: struct), after_struct_column (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: struct), _col4 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: struct), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 389 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/tez/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/tez/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out index 34795b1..c10a210 100644 --- ql/src/test/results/clientpositive/tez/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/tez/schema_evol_orc_nonvec_mapwork_part_all_complex.q.out @@ -667,3 +667,197 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_4] + Select Operator [SEL_3] (rows=4 width=388) + Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=4 width=388) + Output:["_col0","_col1","_col2","_col3","_col4"] + TableScan [TS_0] (rows=4 width=388) + default@part_add_to_various_various_struct5,part_add_to_various_various_struct5,Tbl:COMPLETE,Col:NONE,Output:["insert_num","b","s5","after_struct_column"] + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":905071.0,"c7":-4314.7918,"c8":-1240033819,"c9":"trial ","c10":"trial","c11":"2016-03-07 03:02:22"} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/tez/schema_evol_orc_vec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/tez/schema_evol_orc_vec_mapwork_part_all_complex.q.out index c91eb0b..9aac540 100644 --- ql/src/test/results/clientpositive/tez/schema_evol_orc_vec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/tez/schema_evol_orc_vec_mapwork_part_all_complex.q.out @@ -667,3 +667,197 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_4] + Select Operator [SEL_3] (rows=4 width=388) + Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=4 width=388) + Output:["_col0","_col1","_col2","_col3","_col4"] + TableScan [TS_0] (rows=4 width=388) + default@part_add_to_various_various_struct5,part_add_to_various_various_struct5,Tbl:COMPLETE,Col:NONE,Output:["insert_num","b","s5","after_struct_column"] + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":905071.0,"c7":-4314.7918,"c8":-1240033819,"c9":"trial ","c10":"trial","c11":"2016-03-07 03:02:22"} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/tez/schema_evol_text_nonvec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/tez/schema_evol_text_nonvec_mapwork_part_all_complex.q.out index b0b00b8..49be99f 100644 --- ql/src/test/results/clientpositive/tez/schema_evol_text_nonvec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/tez/schema_evol_text_nonvec_mapwork_part_all_complex.q.out @@ -667,3 +667,197 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_4] + Select Operator [SEL_3] (rows=4 width=97) + Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=4 width=97) + Output:["_col0","_col1","_col2","_col3","_col4"] + TableScan [TS_0] (rows=4 width=97) + default@part_add_to_various_various_struct5,part_add_to_various_various_struct5,Tbl:COMPLETE,Col:NONE,Output:["insert_num","b","s5","after_struct_column"] + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/tez/schema_evol_text_vec_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/tez/schema_evol_text_vec_mapwork_part_all_complex.q.out index 4e03c0b..cc7971d 100644 --- ql/src/test/results/clientpositive/tez/schema_evol_text_vec_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/tez/schema_evol_text_vec_mapwork_part_all_complex.q.out @@ -671,3 +671,197 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_4] + Select Operator [SEL_3] (rows=4 width=97) + Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=4 width=97) + Output:["_col0","_col1","_col2","_col3","_col4"] + TableScan [TS_0] (rows=4 width=97) + default@part_add_to_various_various_struct5,part_add_to_various_various_struct5,Tbl:COMPLETE,Col:NONE,Output:["insert_num","b","s5","after_struct_column"] + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 diff --git ql/src/test/results/clientpositive/tez/schema_evol_text_vecrow_mapwork_part_all_complex.q.out ql/src/test/results/clientpositive/tez/schema_evol_text_vecrow_mapwork_part_all_complex.q.out index f45b517..1312485 100644 --- ql/src/test/results/clientpositive/tez/schema_evol_text_vecrow_mapwork_part_all_complex.q.out +++ ql/src/test/results/clientpositive/tez/schema_evol_text_vecrow_mapwork_part_all_complex.q.out @@ -673,3 +673,197 @@ POSTHOOK: query: drop table part_add_to_various_various_struct4 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_to_various_various_struct4 POSTHOOK: Output: default@part_add_to_various_various_struct4 +PREHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as INNER column +-- +CREATE TABLE part_add_to_various_various_struct5(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) PARTITIONED BY(part INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: CREATE TABLE struct5_a_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_a_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_a.txt' overwrite into table struct5_a_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_a_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_a_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_a_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_a_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_a_txt)struct5_a_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_a_txt.insert_num struct5_a_txt.b struct5_a_txt.s5 struct5_a_txt.after_struct_column +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993} after_original +PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct5 replace columns (insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5 +PREHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: CREATE TABLE struct5_b_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_b_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_b.txt' overwrite into table struct5_b_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_b_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_b_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=2) select * from struct5_b_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_b_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=2 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).after_struct_column SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).b SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).insert_num SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=2).s5 SIMPLE [(struct5_b_txt)struct5_b_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_b_txt.insert_num struct5_b_txt.b struct5_b_txt.s5 struct5_b_txt.after_struct_column +PREHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: CREATE TABLE struct5_c_txt(insert_num int, b STRING, s5 STRUCT, after_struct_column STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@struct5_c_txt +POSTHOOK: query: load data local inpath '../../data/files/struct5_c.txt' overwrite into table struct5_c_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@struct5_c_txt +PREHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +PREHOOK: type: QUERY +PREHOOK: Input: default@struct5_c_txt +PREHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: query: insert into table part_add_to_various_various_struct5 partition(part=1) select * from struct5_c_txt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@struct5_c_txt +POSTHOOK: Output: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).after_struct_column SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:after_struct_column, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).b SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).insert_num SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:insert_num, type:int, comment:null), ] +POSTHOOK: Lineage: part_add_to_various_various_struct5 PARTITION(part=1).s5 SIMPLE [(struct5_c_txt)struct5_c_txt.FieldSchema(name:s5, type:struct, comment:null), ] +struct5_c_txt.insert_num struct5_c_txt.b struct5_c_txt.s5 struct5_c_txt.after_struct_column +PREHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +POSTHOOK: query: explain +select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +Explain +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_4] + Select Operator [SEL_3] (rows=4 width=97) + Output:["_col0","_col1","_col2","_col3","_col4"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=4 width=97) + Output:["_col0","_col1","_col2","_col3","_col4"] + TableScan [TS_0] (rows=4 width=97) + default@part_add_to_various_various_struct5,part_add_to_various_various_struct5,Tbl:COMPLETE,Col:NONE,Output:["insert_num","b","s5","after_struct_column"] + +PREHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +PREHOOK: type: QUERY +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=1 +PREHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +POSTHOOK: query: select insert_num,part,b,s5,after_struct_column from part_add_to_various_various_struct5 order by insert_num +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=1 +POSTHOOK: Input: default@part_add_to_various_various_struct5@part=2 +#### A masked pattern was here #### +insert_num part b s5 after_struct_column +1 1 original {"c1":true,"c2":null,"c3":null,"c4":3244222,"c5":-99999999999,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +2 1 original {"c1":null,"c2":100,"c3":null,"c4":14,"c5":-23866739993,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_original +3 2 new {"c1":true,"c2":null,"c3":null,"c4":-100,"c5":953967041,"c6":62.07915,"c7":718.78,"c8":1,"c9":"verdict ","c10":"verdict","c11":"1987-02-21 19:48:29"} after_new +4 1 new {"c1":false,"c2":-67,"c3":833,"c4":63993,"c5":1255178165,"c6":null,"c7":null,"c8":null,"c9":null,"c10":null,"c11":null} after_new +PREHOOK: query: drop table part_add_to_various_various_struct5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@part_add_to_various_various_struct5 +PREHOOK: Output: default@part_add_to_various_various_struct5 +POSTHOOK: query: drop table part_add_to_various_various_struct5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@part_add_to_various_various_struct5 +POSTHOOK: Output: default@part_add_to_various_various_struct5